{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 采用 xgboost 模型完成商品分类"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 第四步：调整树的参数：subsample 和 colsample_bytree\n",
    "(粗调，参数的步长为0.1；下一步是在粗调最佳参数周围，将步长降为0.05，进行精细调整)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先导入必要的工具包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "\n",
    "import pandas as pd \n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "\n",
    "from sklearn.metrics import log_loss\n",
    "\n",
    "from matplotlib import pyplot\n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###   数据读取\n",
    "注：数据已经提前被处理，只需根据作业要求读取 RentListingInquries_FE_train.csv 文件即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# path to where the data lies\n",
    "path = './data/' \n",
    "fileName = 'RentListingInquries_FE_train.csv'\n",
    "\n",
    "data=pd.read_csv(path+fileName)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target 分布，看看各类样本分布是否均衡"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEKCAYAAADaa8itAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAGlRJREFUeJzt3X+0XWV95/H3hwBK/ZUgV1ZKoNBO2pHaNmqKUWZaRQuBTgUd7cByJGNZk9oBi2s6XWKnU1TE1rZqa0fp0GUKdKyR+otoY9OUio5WfgRFICBDilYiGYiGnzriAN/54zy3nISbe3fiPjk5yfu11l5n7+9+9j7fs+5Kvmvv59nPTlUhSVIfDhh3ApKkfYdFRZLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqzYHjTmBPO+yww+roo48edxqSNFGuv/76b1XV1Fzt9ruicvTRR7Nhw4ZxpyFJEyXJP3Vp5+0vSVJvRlZUkjw5ybVJvpJkY5K3tvglSb6W5Ia2LGnxJHlvkk1JbkzyvKFzrUhye1tWDMWfn+Smdsx7k2RUv0eSNLdR3v56GDihqh5KchDw+SSfbvt+s6o+skP7k4HFbXkBcBHwgiSHAucDS4ECrk+ypqrubW1WAlcDa4HlwKeRJI3FyK5UauChtnlQW2abZ/9U4LJ23NXA/CQLgZOA9VW1rRWS9cDytu/pVfXFGszffxlw2qh+jyRpbiPtU0kyL8kNwD0MCsM1bdeF7RbXe5I8qcWOAO4cOnxzi80W3zxDfKY8VibZkGTD1q1bf+DfJUma2UiLSlU9WlVLgEXAcUmeA7wZ+JfAzwKHAm9qzWfqD6ndiM+Ux8VVtbSqlk5NzTkiTpK0m/bI6K+qug+4ClheVVvaLa6HgT8HjmvNNgNHDh22CLhrjviiGeKSpDEZ5eivqSTz2/ohwMuAr7a+ENpIrdOAm9sha4Az2yiwZcD9VbUFWAecmGRBkgXAicC6tu/BJMvauc4ErhjV75EkzW2Uo78WApcmmcegeF1eVZ9K8vdJphjcvroBeH1rvxY4BdgEfBd4HUBVbUtyAXBda/e2qtrW1n8NuAQ4hMGoL0d+SdIYZTBwav+xdOnS8ol6ae92/J8cP+4U9nlfeMMXdql9kuuraulc7XyiXpLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2xqEiSemNRkST1xqIiSeqNRUWS1BuLiiSpNxYVSVJvLCqSpN5YVCRJvRlZUUny5CTXJvlKko1J3trixyS5JsntST6c5OAWf1Lb3tT2Hz10rje3+G1JThqKL2+xTUnOG9VvkSR1M8orlYeBE6rqZ4AlwPIky4B3Au+pqsXAvcBZrf1ZwL1V9S+A97R2JDkWOB34SWA58P4k85LMA94HnAwcC5zR2kqSxmRkRaUGHmqbB7WlgBOAj7T4pcBpbf3Utk3b/9IkafHVVfVwVX0N2AQc15ZNVXVHVX0fWN3aSpLGZKR9Ku2K4gbgHmA98I/AfVX1SGuyGTiirR8B3AnQ9t8PPHM4vsMxO4tLksZkpEWlqh6tqiXAIgZXFs+eqVn7zE727Wr8CZKsTLIhyYatW7fOnbgkabfskdFfVXUfcBWwDJif5MC2axFwV1vfDBwJ0PY/A9g2HN/hmJ3FZ/r+i6tqaVUtnZqa6uMnSZJmMMrRX1NJ5rf1Q4CXAbcCnwFe1ZqtAK5o62vaNm3/31dVtfjpbXTYMcBi4FrgOmBxG012MIPO/DWj+j2SpLkdOHeT3bYQuLSN0joAuLyqPpXkFmB1krcDXwY+0Np/APiLJJsYXKGcDlBVG5NcDtwCPAKcXVWPAiQ5B1gHzANWVdXGEf4eSdIcRlZUqupG4LkzxO9g0L+yY/x7wKt3cq4LgQtniK8F1v7AyUqSeuET9ZKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2Zs6gkeUqSA9r6jyd5eZKDRp+aJGnSdLlS+Rzw5CRHAFcCrwMuGWVSkqTJ1KWopKq+C7wS+JOqegWDd8JLkrSdTkUlyQuB1wB/3WKjnDJfkjShuhSVNwJvBj7e3m3yowxetCVJ0nbmvOKoqs8Cn03ylLZ9B/Dro05MkjR5uoz+emF7W+Otbftnkrx/5JlJkiZOl9tffwScBHwboKq+AvzcKJOSJE2mTg8/VtWdO4QeHUEukqQJ12UU151JXgRUkoMZ9KfcOtq0JEmTqMuVyuuBs4EjgM3AkrYtSdJ25iwqVfWtqnpNVR1eVc+qqn9fVd+e67gkRyb5TJJbk2xMcm6LvyXJN5Pc0JZTho55c5JNSW5LctJQfHmLbUpy3lD8mCTXJLk9yYfblZQkaUy6jP66NMn8oe0FSVZ1OPcjwG9U1bOBZcDZSaafxH9PVS1py9p23mOB04GfBJYD708yL8k84H3AyQye5D9j6DzvbOdaDNwLnNUhL0nSiHS5/fXTVXXf9EZV3Qs8d66DqmpLVX2prT/IoB/miFkOORVYXVUPV9XXgE3AcW3ZVFV3VNX3gdXAqUkCnAB8pB1/KXBah98jSRqRLkXlgCQLpjeSHMouTtOS5GgGheiaFjonyY1JVg2d+whgeJTZ5hbbWfyZwH1V9cgOcUnSmHQpKu8C/iHJBUkuAP4B+P2uX5DkqcBHgTdW1QPARcCPMejw39LOD5AZDq/diM+Uw8okG5Js2Lp1a9fUJUm7qEtH/WXAq4C7gXuAV1bVX3Q5eXvvykeBD1bVx9r57q6qR6vqMeDPGNzegsGVxpFDhy8C7pol/i1gfpIDd4jP9BsurqqlVbV0amqqS+qSpN3Q9c2PXwU+BlwBPJTkqLkOaH0eHwBurap3D8UXDjV7BXBzW18DnJ7kSUmOARYD1wLXAYvbSK+DGXTmr6mqYjCx5ava8StafpKkMZmzbyTJG4DzGVypPMrgtlMBPz3HoccDrwVuSnJDi/0Wg9FbS9o5vg78KkCbAfly4BYGI8fOrqpHWw7nAOuAecCqqtrYzvcmYHWStwNfZlDEJElj0qXD/VzgJ7o8mzKsqj7PzP0ea2c55kLgwhnia2c6rs2YfNyOcUnSeHS5/XUncP+oE5EkTb4uVyp3AFcl+Wvg4engcD+JJEnQrah8oy0Ht0WSpBl1efPjWwGSPKWqvjP6lCRJk8o3P0qSeuObHyVJvfHNj5Kk3vjmR0lSb3zzoySpN7NeqbQXZL22ql6zh/KRJE2wWa9U2txbp+6hXCRJE65Ln8oXkvx34MPAPz+nMv1WR0mSpnUpKi9qn28bihWDV/lKkvTP5upTOQC4qKou30P5SJIm2Fx9Ko8B5+yhXCRJE67LkOL1Sf5LkiOTHDq9jDwzSdLE6dKn8ivtc/jZlAJ+tP90JEmTrMssxcfsiUQkSZOvyzvqz5wpXlWX9Z+OJGmSdbn99bND608GXgp8CbCoSJK20+X21xuGt5M8A/iLkWUkSZpYnaa+38F3gcVzNWqjxT6T5NYkG5Oc2+KHJlmf5Pb2uaDFk+S9STYluTHJ84bOtaK1vz3JiqH485Pc1I55b5Lsxu+RJPWky5sfP5lkTVs+BdwGXNHh3I8Av1FVzwaWAWcnORY4D7iyqhYDV7ZtgJMZFKvFwErgovb9hwLnAy8AjgPOny5Erc3KoeOWd8hLkjQiXfpU/nBo/RHgn6pq81wHVdUWYEtbfzDJrQymzz8VeHFrdilwFfCmFr+sqgq4Osn8JAtb2/VVtQ0gyXpgeZKrgKdX1Rdb/DLgNODTHX6TJGkEuhSVbwBbqup7AEkOSXJ0VX2965ckORp4LnANcHgrOFTVliTPas2OAIbfMLm5xWaLb54hLkkaky59Kn8FPDa0/WiLdZLkqcBHgTdW1QOzNZ0hVrsRnymHlUk2JNmwdevWuVKWJO2mLkXlwKr6/vRGWz+4y8mTHMSgoHywqj7Wwne321q0z3tafDNw5NDhi4C75ogvmiH+BFV1cVUtraqlU1NTXVKXJO2GLkVla5KXT28kORX41lwHtZFYHwBurap3D+1aA0yP4FrB453+a4Az2yiwZcD97TbZOuDEJAtaB/2JwLq278Eky9p3nUm3AQSSpBHp0qfyeuCD7UVdMLhCmPEp+x0cD7wWuCnJDS32W8DvAZcnOYtBf82r2761wCnAJgbDll8HUFXbklwAXNfavW260x74NeAS4BAGHfR20kvSGHV5+PEfgWWtbyRV9WCXE1fV55m53wMGT+Xv2L7YftLK4X2rgFUzxDcAz+mSjyRp9Lo8p/KOJPOr6qE2NHhBkrfvieQkSZOlS5/KyVV13/RGVd3L4DaVJEnb6VJU5iV50vRGkkOAJ83SXpK0n+rSUf8/gSuT/DmD50B+hcGT8JIkbadLR/3vJ7kReFkLXVBV60abliRpEnW5UgH4MnAQgyuVL48uHUnSJOsy+uuXgWuBVwG/DFyT5FWjTkySNHm6XKn8V+Bnq+oegCRTwN8BHxllYpKkydNl9NcB0wWl+XbH4yRJ+5kuVyp/k2Qd8KG2/e8YTKkiSdJ2uoz++s0krwT+FYNpVy6uqo+PPDNJ0sTpNPqrTVv/sTkbSpL2a/aNSJJ6Y1GRJPVmp0UlyZXt8517Lh1J0iSbrU9lYZKfB16eZDU7vBulqr400swkSRNntqLyO8B5DN79/u4d9hVwwqiSkiRNpp0Wlar6CPCRJP+tqi7YgzlJkiZUl+dULkjycuDnWuiqqvrUaNOSJE2iLhNK/i5wLnBLW85tMUmSttPl4cdfBJZU1WMASS5lMP39m0eZmCRp8nR9TmX+0PozRpGIJGnydSkqvwt8Ockl7SrleuAdcx2UZFWSe5LcPBR7S5JvJrmhLacM7Xtzkk1Jbkty0lB8eYttSnLeUPyYJNckuT3Jh5Mc3PVHS5JGY86iUlUfApYxmPvrY8ALq2p1h3NfAiyfIf6eqlrSlrUASY4FTgd+sh3z/iTzkswD3gecDBwLnNHaAryznWsxcC9wVoecJEkj1On2V1Vtqao1VXVFVf2fjsd8DtjWMY9TgdVV9XBVfQ3YBBzXlk1VdUdVfR9YDZyaJAyek5l+UdilwGkdv0uSNCLjmPvrnCQ3tttjC1rsCODOoTabW2xn8WcC91XVIzvEZ5RkZZINSTZs3bq1r98hSdrBni4qFwE/BiwBtgDvavHM0LZ2Iz6jqrq4qpZW1dKpqaldy1iS1NmsRSXJAcMd7T+oqrq7qh5tw5P/jMHtLRhcaRw51HQRcNcs8W8B85McuENckjRGsxaV9p//V5Ic1ceXJVk4tPkKYLpgrQFOT/KkJMcAi4FrgeuAxW2k18EMOvPXVFUBnwFe1Y5fAVzRR46SpN3X5eHHhcDGJNcC35kOVtXLZzsoyYeAFwOHJdkMnA+8OMkSBreqvg78ajvXxiSXM3hi/xHg7Kp6tJ3nHGAdMA9YVVUb21e8CVid5O0MHsb8QJcfLEkanS5F5a27c+KqOmOG8E7/46+qC4ELZ4ivBdbOEL+Dx2+fSZL2Al0mlPxskh8BFlfV3yX5IQZXDZIkbafLhJL/kcHzIP+jhY4APjHKpCRJk6nLkOKzgeOBBwCq6nbgWaNMSpI0mboUlYfb0+wAtGG8O30mRJK0/+pSVD6b5LeAQ5L8AvBXwCdHm5YkaRJ1KSrnAVuBmxgMAV4L/PYok5IkTaYuo78ea1PeX8Pgttdt7eFDSZK2M2dRSfKLwJ8C/8hgzq1jkvxqVX161MlJkiZLl4cf3wW8pKo2AST5MeCvAYuKJGk7XfpU7pkuKM0dwD0jykeSNMF2eqWS5JVtdWOStcDlDPpUXs1gokdJkrYz2+2vXxpavxv4+ba+FVjwxOaSpP3dTotKVb1uTyYiSZp8XUZ/HQO8ATh6uP1cU99LkvY/XUZ/fYLBlPWfBB4bbTqSpEnWpah8r6reO/JMJEkTr0tR+eMk5wN/Czw8HayqL40sK0nSROpSVH4KeC1wAo/f/qq2Le2VvvG2nxp3CvuFo37npnGnoL1Ml6LyCuBHh6e/lyRpJl2eqP8KMH/UiUiSJl+XK5XDga8muY7t+1QcUixJ2k6XonL+7pw4ySrg3zCYO+w5LXYo8GEGz7x8Hfjlqro3SYA/Bk4Bvgv8h+mBAElW8Pj7W95eVZe2+POBS4BDGLzj5Vyn5Jek8Zrz9ldVfXampcO5LwGW7xA7D7iyqhYDV7ZtgJOBxW1ZCVwE/1yEzgdeABwHnJ9keoqYi1rb6eN2/C5J0h42Z1FJ8mCSB9ryvSSPJnlgruOq6nPAth3CpwKXtvVLgdOG4pfVwNXA/CQLgZOA9VW1raruBdYDy9u+p1fVF9vVyWVD55IkjUmXNz8+bXg7yWkMrhp2x+FVtaWdd0uSZ7X4EcCdQ+02t9hs8c0zxGeUZCWDqxqOOuqo3UxdkjSXLqO/tlNVn6D/Z1Qy01ftRnxGVXVxVS2tqqVTU1O7maIkaS5dJpR85dDmAcBSZvkPfA53J1nYrlIW8vjLvjYDRw61WwTc1eIv3iF+VYsvmqG9JGmMulyp/NLQchLwIIM+kN2xBljR1lcAVwzFz8zAMuD+dptsHXBikgWtg/5EYF3b92CSZW3k2JlD55IkjUmXPpXdeq9Kkg8xuMo4LMlmBqO4fg+4PMlZwDcYvEUSBkOCTwE2MRhS/Lr23duSXMDjb5p8W1VNd/7/Go8PKf50WyRJYzTb64R/Z5bjqqoumO3EVXXGTna9dKaTAWfv5DyrgFUzxDcAz5ktB0nSnjXblcp3Zog9BTgLeCYwa1GRJO1/Znud8Lum15M8DTiXwW2p1cC7dnacJGn/NWufSnui/T8Dr2HwsOLz2kOIkiQ9wWx9Kn8AvBK4GPipqnpoj2UlSZpIsw0p/g3ghxlM5njX0FQtD3aZpkWStP+ZrU9ll5+2lyTt3ywckqTeWFQkSb2xqEiSemNRkST1xqIiSeqNRUWS1BuLiiSpNxYVSVJvLCqSpN5YVCRJvbGoSJJ6Y1GRJPXGoiJJ6o1FRZLUG4uKJKk3YykqSb6e5KYkNyTZ0GKHJlmf5Pb2uaDFk+S9STYluTHJ84bOs6K1vz3JinH8FknS48Z5pfKSqlpSVUvb9nnAlVW1GLiybQOcDCxuy0rgIhgUIeB84AXAccD504VIkjQee9Ptr1OBS9v6pcBpQ/HLauBqYH6ShcBJwPqq2lZV9wLrgeV7OmlJ0uPGVVQK+Nsk1ydZ2WKHV9UWgPb5rBY/Arhz6NjNLbazuCRpTHb6jvoRO76q7kryLGB9kq/O0jYzxGqW+BNPMChcKwGOOuqoXc1VktTRWK5Uququ9nkP8HEGfSJ3t9tatM97WvPNwJFDhy8C7polPtP3XVxVS6tq6dTUVJ8/RZI0ZI8XlSRPSfK06XXgROBmYA0wPYJrBXBFW18DnNlGgS0D7m+3x9YBJyZZ0DroT2wxSdKYjOP21+HAx5NMf/9fVtXfJLkOuDzJWcA3gFe39muBU4BNwHeB1wFU1bYkFwDXtXZvq6pte+5nSJJ2tMeLSlXdAfzMDPFvAy+dIV7A2Ts51ypgVd85SpJ2z940pFiSNOEsKpKk3oxrSPFEeP5vXjbuFPZ51//BmeNOQVKPvFKRJPXGoiJJ6o1FRZLUG4uKJKk3FhVJUm8sKpKk3lhUJEm9sahIknpjUZEk9caiIknqjUVFktQbi4okqTcWFUlSbywqkqTeWFQkSb2xqEiSemNRkST1xqIiSerNxBeVJMuT3JZkU5Lzxp2PJO3PJrqoJJkHvA84GTgWOCPJsePNSpL2XxNdVIDjgE1VdUdVfR9YDZw65pwkab816UXlCODOoe3NLSZJGoMDx53ADygzxOoJjZKVwMq2+VCS20aa1XgdBnxr3El0lT9cMe4U9iYT9bcD4PyZ/gnutybq75df3+W/3Y90aTTpRWUzcOTQ9iLgrh0bVdXFwMV7KqlxSrKhqpaOOw/tOv92k82/38Ck3/66Dlic5JgkBwOnA2vGnJMk7bcm+kqlqh5Jcg6wDpgHrKqqjWNOS5L2WxNdVACqai2wdtx57EX2i9t8+yj/dpPNvx+Qqif0a0uStFsmvU9FkrQXsajsI5yuZnIlWZXkniQ3jzsX7ZokRyb5TJJbk2xMcu64cxo3b3/tA9p0Nf8b+AUGw6yvA86oqlvGmpg6SfJzwEPAZVX1nHHno+6SLAQWVtWXkjwNuB44bX/+t+eVyr7B6WomWFV9Dtg27jy066pqS1V9qa0/CNzKfj6rh0Vl3+B0NdKYJTkaeC5wzXgzGS+Lyr6h03Q1kkYjyVOBjwJvrKoHxp3POFlU9g2dpquR1L8kBzEoKB+sqo+NO59xs6jsG5yuRhqDJAE+ANxaVe8edz57A4vKPqCqHgGmp6u5Fbjc6WomR5IPAV8EfiLJ5iRnjTsndXY88FrghCQ3tOWUcSc1Tg4pliT1xisVSVJvLCqSpN5YVCRJvbGoSJJ6Y1GRJPXGoiL1KMn8JP9pD3zPi5O8aNTfI+0qi4rUr/lA56KSgd35d/hiwKKivY7PqUg9SjI9Q/RtwGeAnwYWAAcBv11VV7SJBz/d9r8QOA14GfAmBtPr3A48XFXnJJkC/hQ4qn3FG4FvAlcDjwJbgTdU1f/aE79PmotFRepRKxifqqrnJDkQ+KGqeiDJYQwKwWLgR4A7gBdV1dVJfhj4B+B5wIPA3wNfaUXlL4H3V9XnkxwFrKuqZyd5C/BQVf3hnv6N0mwOHHcC0j4swDvaS7geY/A6gsPbvn+qqqvb+nHAZ6tqG0CSvwJ+vO17GXDsYIopAJ7eXgYl7ZUsKtLovAaYAp5fVf8vydeBJ7d93xlqN9OrC6YdALywqv7vcHCoyEh7FTvqpX49CExfSTwDuKcVlJcwuO01k2uBn0+yoN0y+7dD+/6WwWShACRZMsP3SHsNi4rUo6r6NvCFJDcDS4ClSTYwuGr56k6O+SbwDgZvDPw74Bbg/rb719s5bkxyC/D6Fv8k8Io2K+6/HtkPknaRHfXSXiDJU6vqoXal8nFgVVV9fNx5SbvKKxVp7/CWJDcANwNfAz4x5nyk3eKViiSpN16pSJJ6Y1GRJPXGoiJJ6o1FRZLUG4uKJKk3FhVJUm/+P7RyK1qmT6eeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(data.interest_level);\n",
    "pyplot.xlabel('target');\n",
    "pyplot.ylabel('Number of occurrences');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 从原始数据中分离输入特征x和输出y\n",
    "X = data.drop('interest_level', axis = 1)\n",
    "y = data['interest_level']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = X.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将数据分割训练数据与测试数据\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# 随机采样20%的数据构建测试样本，其余作为训练样本\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3,random_state = 0)\n",
    "#print('X_train.shape: %s %s'%X_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "各类样本不均衡，交叉验证是采用StratifiedKFold，在每折采样时各类样本按比例采样"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# prepare cross validation\n",
    "kfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'subsample': [0.3, 0.4, 0.5, 0.6, 0.7, 0.8],\n",
       " 'colsample_bytree': [0.6, 0.7, 0.8, 0.9]}"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#max_depth 建议3-10， min_child_weight=1／sqrt(ratio_rare_event) =5.5\n",
    "subsample = [i/10.0 for i in range(3,9)]\n",
    "colsample_bytree = [i/10.0 for i in range(6,10)]\n",
    "param_test3_1 = dict(subsample=subsample, colsample_bytree=colsample_bytree)\n",
    "param_test3_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/model_selection/_search.py:761: DeprecationWarning: The grid_scores_ attribute was deprecated in version 0.18 in favor of the more elaborate cv_results_ attribute. The grid_scores_ attribute will not be available from 0.20\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "([mean: -0.59919, std: 0.00213, params: {'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  mean: -0.59619, std: 0.00325, params: {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  mean: -0.59381, std: 0.00296, params: {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  mean: -0.59353, std: 0.00319, params: {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  mean: -0.59358, std: 0.00373, params: {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  mean: -0.59230, std: 0.00317, params: {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  mean: -0.59696, std: 0.00338, params: {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  mean: -0.59534, std: 0.00262, params: {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  mean: -0.59374, std: 0.00232, params: {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  mean: -0.59310, std: 0.00304, params: {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  mean: -0.59260, std: 0.00275, params: {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  mean: -0.59299, std: 0.00309, params: {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  mean: -0.59740, std: 0.00366, params: {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  mean: -0.59563, std: 0.00326, params: {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  mean: -0.59390, std: 0.00352, params: {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  mean: -0.59360, std: 0.00366, params: {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  mean: -0.59344, std: 0.00353, params: {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  mean: -0.59219, std: 0.00266, params: {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  mean: -0.59735, std: 0.00341, params: {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  mean: -0.59572, std: 0.00411, params: {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  mean: -0.59464, std: 0.00263, params: {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  mean: -0.59331, std: 0.00239, params: {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  mean: -0.59255, std: 0.00363, params: {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  mean: -0.59250, std: 0.00278, params: {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       " -0.5921903138953263)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb3_1 = XGBClassifier(\n",
    "        learning_rate =0.1,\n",
    "        n_estimators=187,  #第二轮参数调整得到的n_estimators最优值\n",
    "        max_depth=5,\n",
    "        min_child_weight=1,\n",
    "        gamma=0,\n",
    "        subsample=0.3,\n",
    "        colsample_bytree=0.8,\n",
    "        colsample_bylevel = 0.7,\n",
    "        objective= 'multi:softprob',\n",
    "        seed=3)\n",
    "\n",
    "\n",
    "gsearch3_1 = GridSearchCV(xgb3_1, param_grid = param_test3_1, scoring='neg_log_loss',n_jobs=-1, cv=kfold)\n",
    "gsearch3_1.fit(X_train , y_train)\n",
    "\n",
    "gsearch3_1.grid_scores_, gsearch3_1.best_params_,     gsearch3_1.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([ 72.42257929,  85.92545481,  99.32703166, 101.9432714 ,\n",
       "         99.38955164,  97.63540883,  97.74130154, 111.2358108 ,\n",
       "        118.84883118, 119.94031277, 116.87396774, 114.06594319,\n",
       "        112.5326159 , 129.29172587, 140.48642378, 139.7844058 ,\n",
       "        136.21708155, 130.41596909, 126.77695117, 143.15755777,\n",
       "        154.95204763, 159.18710122, 152.50580716, 135.53883443]),\n",
       " 'std_fit_time': array([0.50118416, 3.40350983, 1.11513809, 0.62976196, 1.39751824,\n",
       "        2.48708403, 1.57848864, 0.66838793, 0.76969805, 1.27596417,\n",
       "        1.21963443, 1.59118758, 1.98235502, 1.25641544, 2.06924754,\n",
       "        1.84820179, 2.88204965, 2.38542319, 1.39341162, 2.93956929,\n",
       "        1.52614056, 2.68375353, 2.67092662, 7.0457565 ]),\n",
       " 'mean_score_time': array([0.42277875, 0.46232681, 0.45002637, 0.48669577, 0.43330984,\n",
       "        0.43663092, 0.4358861 , 0.43396072, 0.47144499, 0.45667377,\n",
       "        0.45224562, 0.49724865, 0.47620082, 0.42500186, 0.49548588,\n",
       "        0.4758018 , 0.47250633, 0.49520888, 0.47553682, 0.4570344 ,\n",
       "        0.46204486, 0.45558667, 0.43400455, 0.31252279]),\n",
       " 'std_score_time': array([0.01719389, 0.0299435 , 0.02850827, 0.05863851, 0.01540532,\n",
       "        0.01572718, 0.01946068, 0.0215612 , 0.07522627, 0.02922685,\n",
       "        0.03553854, 0.07472099, 0.0288769 , 0.01528686, 0.07479131,\n",
       "        0.05085343, 0.01729049, 0.03368484, 0.03280718, 0.01634577,\n",
       "        0.04061361, 0.01864732, 0.07894286, 0.02786022]),\n",
       " 'param_colsample_bytree': masked_array(data=[0.6, 0.6, 0.6, 0.6, 0.6, 0.6, 0.7, 0.7, 0.7, 0.7, 0.7,\n",
       "                    0.7, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.9, 0.9, 0.9, 0.9,\n",
       "                    0.9, 0.9],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_subsample': masked_array(data=[0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6, 0.7,\n",
       "                    0.8, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.3, 0.4, 0.5, 0.6,\n",
       "                    0.7, 0.8],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'colsample_bytree': 0.6, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.6, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.7, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.8, 'subsample': 0.8},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.3},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.4},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.5},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.6},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.7},\n",
       "  {'colsample_bytree': 0.9, 'subsample': 0.8}],\n",
       " 'split0_test_score': array([-0.60168861, -0.59937092, -0.59714892, -0.59861458, -0.59905966,\n",
       "        -0.59632324, -0.60082165, -0.59863917, -0.5962412 , -0.59565287,\n",
       "        -0.5953705 , -0.59674347, -0.60278146, -0.59954944, -0.59775346,\n",
       "        -0.5987747 , -0.59813658, -0.59593393, -0.60105701, -0.60007326,\n",
       "        -0.59706848, -0.59560236, -0.59724032, -0.59567262]),\n",
       " 'split1_test_score': array([-0.59681452, -0.59159378, -0.59082629, -0.59091109, -0.59007992,\n",
       "        -0.5879623 , -0.59245404, -0.59238686, -0.59074099, -0.59062666,\n",
       "        -0.58960191, -0.58833854, -0.59416526, -0.5910625 , -0.58927   ,\n",
       "        -0.59040199, -0.5895077 , -0.58877861, -0.59153179, -0.5907464 ,\n",
       "        -0.59143849, -0.59052275, -0.58799725, -0.58905598]),\n",
       " 'split2_test_score': array([-0.5965357 , -0.59420916, -0.5915181 , -0.59101722, -0.59196719,\n",
       "        -0.59081519, -0.59378271, -0.59287066, -0.59160765, -0.58970645,\n",
       "        -0.59070517, -0.59209907, -0.59355476, -0.59317917, -0.59230547,\n",
       "        -0.58966131, -0.59136978, -0.58970122, -0.59668608, -0.59255517,\n",
       "        -0.5926354 , -0.59034136, -0.59039351, -0.59066369]),\n",
       " 'split3_test_score': array([-0.59991685, -0.59545265, -0.59192313, -0.59111422, -0.58988725,\n",
       "        -0.59080226, -0.59737001, -0.59462169, -0.59370359, -0.59189865,\n",
       "        -0.59089303, -0.59169487, -0.59586417, -0.59542301, -0.59189877,\n",
       "        -0.59208868, -0.59096353, -0.59250382, -0.5969883 , -0.59420564,\n",
       "        -0.59374757, -0.59445821, -0.590688  , -0.59119994]),\n",
       " 'split4_test_score': array([-0.60099359, -0.60033092, -0.59765848, -0.5960048 , -0.59692992,\n",
       "        -0.59559953, -0.60036715, -0.5981795 , -0.59642821, -0.59763107,\n",
       "        -0.59644005, -0.59607541, -0.60062688, -0.59894662, -0.59825546,\n",
       "        -0.5970809 , -0.59723473, -0.59403421, -0.60050566, -0.60103847,\n",
       "        -0.59831303, -0.59564398, -0.59641701, -0.5959272 ]),\n",
       " 'mean_test_score': array([-0.59918981, -0.59619133, -0.59381488, -0.59353238, -0.59358475,\n",
       "        -0.5923004 , -0.59695899, -0.5953395 , -0.59374423, -0.59310301,\n",
       "        -0.59260201, -0.59299016, -0.59739848, -0.59563203, -0.59389648,\n",
       "        -0.59360147, -0.59344238, -0.59219031, -0.59735361, -0.59572361,\n",
       "        -0.59464047, -0.59331365, -0.59254711, -0.59250378]),\n",
       " 'std_test_score': array([0.00213134, 0.00325142, 0.00295548, 0.00319339, 0.00373468,\n",
       "        0.00317298, 0.00337904, 0.0026185 , 0.00232487, 0.00303717,\n",
       "        0.00275367, 0.00308927, 0.00366015, 0.00326422, 0.0035158 ,\n",
       "        0.00365843, 0.00353103, 0.00265896, 0.00340961, 0.00410567,\n",
       "        0.00262498, 0.00239176, 0.00362767, 0.00278334]),\n",
       " 'rank_test_score': array([24, 20, 14, 10, 11,  2, 21, 17, 13,  7,  5,  6, 23, 18, 15, 12,  9,\n",
       "         1, 22, 19, 16,  8,  4,  3], dtype=int32),\n",
       " 'split0_train_score': array([-0.51433899, -0.51062813, -0.50844814, -0.50631376, -0.50681903,\n",
       "        -0.50617572, -0.50962657, -0.5071274 , -0.50500836, -0.50409169,\n",
       "        -0.50420155, -0.50439327, -0.50845671, -0.50450153, -0.50231996,\n",
       "        -0.50127591, -0.50098843, -0.50164501, -0.50475649, -0.50004132,\n",
       "        -0.49935542, -0.49917437, -0.49922408, -0.49918183]),\n",
       " 'split1_train_score': array([-0.51581363, -0.51134954, -0.50973258, -0.50905212, -0.50955846,\n",
       "        -0.50825151, -0.51069671, -0.50667079, -0.50516522, -0.50310378,\n",
       "        -0.50451615, -0.5055567 , -0.50807877, -0.50386097, -0.50106094,\n",
       "        -0.50152489, -0.5018931 , -0.50280609, -0.50487418, -0.50147351,\n",
       "        -0.50086085, -0.50182148, -0.49972576, -0.50021482]),\n",
       " 'split2_train_score': array([-0.51477048, -0.51227154, -0.51117976, -0.50831578, -0.50845437,\n",
       "        -0.50726946, -0.51142801, -0.5082454 , -0.50576092, -0.50567469,\n",
       "        -0.50682397, -0.50640661, -0.50901464, -0.50529754, -0.50469803,\n",
       "        -0.50347459, -0.50249261, -0.50336411, -0.5074874 , -0.50392782,\n",
       "        -0.50134943, -0.50014393, -0.50089429, -0.50152807]),\n",
       " 'split3_train_score': array([-0.5153354 , -0.51198095, -0.51002308, -0.50998994, -0.50958298,\n",
       "        -0.50953408, -0.51090953, -0.50853035, -0.50624555, -0.50465558,\n",
       "        -0.50601464, -0.50523866, -0.50946109, -0.50389692, -0.50384157,\n",
       "        -0.50251222, -0.5034196 , -0.50423846, -0.50533514, -0.50256965,\n",
       "        -0.5011185 , -0.50099871, -0.50095149, -0.5024525 ]),\n",
       " 'split4_train_score': array([-0.51381722, -0.50941245, -0.50672776, -0.50636966, -0.50546981,\n",
       "        -0.50529054, -0.50991701, -0.50547284, -0.50357285, -0.5024698 ,\n",
       "        -0.50243687, -0.50207021, -0.50708036, -0.50359413, -0.50209126,\n",
       "        -0.50153651, -0.49999231, -0.50059187, -0.50558708, -0.50209734,\n",
       "        -0.50042955, -0.49870834, -0.49855599, -0.49876928]),\n",
       " 'mean_train_score': array([-0.51481515, -0.51112852, -0.50922226, -0.50800825, -0.50797693,\n",
       "        -0.50730426, -0.51051557, -0.50720936, -0.50515058, -0.50399911,\n",
       "        -0.50479864, -0.50473309, -0.50841832, -0.50423022, -0.50280235,\n",
       "        -0.50206482, -0.50175721, -0.50252911, -0.50560806, -0.50202193,\n",
       "        -0.50062275, -0.50016937, -0.49987032, -0.5004293 ]),\n",
       " 'std_train_score': array([0.00070601, 0.00102735, 0.00152022, 0.00146066, 0.00160841,\n",
       "        0.00149661, 0.00065866, 0.00110731, 0.00090345, 0.00113026,\n",
       "        0.00152282, 0.00147926, 0.00081852, 0.00061071, 0.00129957,\n",
       "        0.00082227, 0.00118564, 0.001283  , 0.00098698, 0.00127763,\n",
       "        0.00070342, 0.0011439 , 0.00093632, 0.00138985])}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gsearch3_1.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('mean_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('std_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split0_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split1_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split2_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split3_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n",
      "/root/anaconda3/lib/python3.6/site-packages/sklearn/utils/deprecation.py:122: FutureWarning: You are accessing a training score ('split4_train_score'), which will not be available by default any more in 0.21. If you need training scores, please set return_train_score=True\n",
      "  warnings.warn(*warn_args, **warn_kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: -0.592190 using {'colsample_bytree': 0.8, 'subsample': 0.8}\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEKCAYAAAA4t9PUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3XdUFFcbwOHfpYsi0lRUULGjUqyxBzv2FlssWKLGGFvsxmiMJhoTe2JvXyzYYq9JNGrUWKKIBQuWCFYEBAtIu98fuxLQRVZhAeE+5+w57MydmXf2wL7MrUJKiaIoiqK8K6PMDkBRFEV5v6lEoiiKoqSJSiSKoihKmqhEoiiKoqSJSiSKoihKmqhEoiiKoqSJSiSKoihKmqhEoiiKoqSJSiSKoihKmphkdgAZwd7eXhYrViyzw1AURXmv/PPPP4+klA6plcsRiaRYsWKcPn06s8NQFEV5rwgh/tWnnKraUhRFUdJEJRJFURQlTVQiURRFUdIkR7SRKEpWEBsbS3BwMNHR0ZkdiqIkY2FhQZEiRTA1NX2n41UiUZQMEhwcjJWVFcWKFUMIkdnhKAoAUkpCQ0MJDg6mePHi73QOVbWlKBkkOjoaOzs7lUSULEUIgZ2dXZqelFUiUZQMpJKIkhWl9fdSJZI3+CPgAb4nb2d2GIqiKFmaaiNJgZSSdSdvc/jqI9yK5MO1UN7MDklRFCVLUk8kKRBCML29G9aWpgz2PUtUTHxmh6Qoafb48WN+/vnndzp29uzZPH/+PN1iWblyJYMGDUq38+nrzz//pEWLFm993NvGe+vWLdauXfvW19FXWFgYjRo1olSpUjRq1Ijw8HCd5W7fvk3jxo0pV64crq6u3Lp1K91jMWgiEUI0FUJcEUIECiHG6NjvI4QIEUL4aV99k+ybLoS4oH11SrK9vhDijHb7KiGEwZ6q7PKYM7OjO4EPnzJl1yVDXUZRMkxWSiTZ3ZsSSVxcXJrPP23aNBo0aMC1a9do0KAB06ZN01muR48ejBw5koCAAE6ePEn+/PnTfO1XGexLWAhhDPwENAKCgVNCiO1Syle/kddLKQe9cmxzoBLgAZgDh4QQe4CnwCqggZTyqhBiMtATWGao+6hTyoF+dV1YfPgGdUs70KR8QUNdSslBvt5xkUt3I9P1nK6F8jKxZfk3lhkzZgzXr1/Hw8ODRo0akT9/fjZs2MCLFy9o27YtX3/9Nc+ePaNjx44EBwcTHx/PhAkTePDgAXfv3sXLywt7e3sOHjyo8/x79+5l3LhxxMfHY29vzx9//EFYWBi9e/fmxo0bWFpasnjxYtzc3JIdt3HjRr7++muMjY2xtrbm8OHD3Lp1i+7du/Ps2TMA5s+fT82aNfnzzz+ZOHEiBQoUwM/Pj3bt2lGxYkXmzJlDVFQUW7dupUSJEvj4+GBhYcHFixd58OABM2fOfO1J5NmzZ3z++eecP3+euLg4Jk2aROvWrVP8/IKCgmjatCk3b96ka9euTJw4kQkTJmBvb8+QIUMAGD9+PAUKFGDt2rUEBATg4eFBz549sbGxYdeuXURHR/Ps2TMOHDjAjBkzXvv8AVavXs3cuXOJiYmhevXq/PzzzxgbGyeLZdu2bfz5558A9OzZkw8//JDp06cnK3Pp0iXi4uJo1KgRAHny5HnTr8c7M2QbSTUgUEp5A0AI4Qu0BvT5194VOCSljAPihBDngKbAQeCFlPKqttxvwFgMmEgARjQuw7Hrjxi92R/3IvkoaG1hyMspisFMmzaNCxcu4Ofnx/79+9m0aRMnT55ESkmrVq04fPgwISEhFCpUiF27dgEQERGBtbU1M2fO5ODBg9jb2+s8d0hICJ988gmHDx+mePHihIWFATBx4kQ8PT3ZunUrBw4coEePHvj5+SU7dvLkyezbt4/ChQvz+PFjAPLnz89vv/2GhYUF165do0uXLomTr547d46AgABsbW1xcXGhb9++nDx5kjlz5jBv3jxmz54NaJ4KDh06xPXr1/Hy8iIwMDDZdadOnUr9+vVZvnw5jx8/plq1ajRs2JDcuXPrvMeTJ09y4cIFLC0tqVq1Ks2bN6dPnz60a9eOIUOGkJCQgK+vLydPnsTNzY0ffviBnTt3ApqqsePHj+Pv74+trS379+/n2rVrr33+Dg4OrF+/nqNHj2JqasrAgQNZs2YNPXr0oG/fvgwYMIAqVarw4MEDHB0dAXB0dOThw4evxXv16lXy5ctHu3btuHnzJg0bNmTatGmvJaW0MmQiKQwEJXkfDFTXUa69EKIucBUYJqUMAs4BE4UQMwFLwAtNAnoEmAohqkgpTwMdACcD3gMAZiZGzOnsSYu5fzFsvR+r+1bH2Eh141TeXWpPDhlh//797N+/H09PTwCePn3KtWvXqFOnDiNGjGD06NG0aNGCOnXq6HW+v//+m7p16yYOarO1tQXgr7/+YvPmzQDUr1+f0NBQIiIikh1bq1YtfHx86NixI+3atQM0MwEMGjQIPz8/jI2NuXr1amL5qlWrJn6JlihRgsaNGwNQsWLFZE9LHTt2xMjIiFKlSuHi4sLly5df+wy2b9/ODz/8AGjG+ty+fZty5crpvMdGjRphZ2cHQLt27fjrr78YOnQodnZ2nD17lgcPHuDp6ZlYRtfxLz+XlD5/f39//vnnH6pWrQpAVFRUYnXU0qVLdZ43JXFxcRw5coSzZ8/i7OxMp06dWLlyJX369Hmr86TGkIlE1zetfOX9DmCdlPKFEGIAmmqr+lLK/UKIqsAxIAQ4DsRJKaUQojMwSwhhDuwHdFY2CiH6Af0AnJ2d03wzJRzyMKmVK6M3n2fx4Rt8+mGJNJ9TUTKTlJKxY8fSv3//1/b9888/7N69m7Fjx9K4cWO++uorvc6nazyClK/+2b8+bmHhwoWcOHGCXbt24eHhgZ+fH/PmzaNAgQKcO3eOhIQELCz+qwkwNzdP/NnIyCjxvZGRUbL2h1ev8+p7KSWbN2+mTJkyqd7fm87Xt29fVq5cyf379+ndu3eKxyd90knp8583bx49e/bku+++e2MsBQoU4N69ezg6OnLv3j2dbR9FihTB09MTFxcXANq0acPff/+d7onEkI3twSR/WigC3E1aQEoZKqV8oX27BKicZN9UKaWHlLIRmqR0Tbv9uJSyjpSyGnD45fZXSSkXSymrSCmrODikui6LXjpWcaJZxYL8uP8K54Iep8s5FSUjWVlZ8eTJEwCaNGnC8uXLefr0KQB37tzh4cOH3L17F0tLS7p168aIESM4c+bMa8fqUqNGDQ4dOsTNmzcBEqu26taty5o1awBNjyl7e3vy5k3enf769etUr16dyZMnY29vT1BQEBERETg6OmJkZMQvv/xCfPzb95zcuHEjCQkJXL9+nRs3bryWMJo0acK8efMSk93Zs2ffeL7ffvuNsLCwxLaYWrVqAdC2bVv27t3LqVOnaNKkCZD655XS59+gQQM2bdqUWFUVFhbGv/++vixIq1atWLVqFQCrVq3S2bZTtWpVwsPDCQkJAeDAgQO4urq+8R7fhSGfSE4BpYQQxYE7QGega9ICQghHKeU97dtWQIB2uzGQT0oZKoRwA9zQPH0ghMgvpXyofSIZDUw14D0kI4Tgu7Zu+N0+zBDfs+wcXIc85moojvL+sLOzo1atWlSoUAFvb2+6du1KjRo1AE1D7OrVqwkMDGTkyJEYGRlhamrKggULAOjXrx/e3t44OjrqbGx3cHBg8eLFtGvXjoSEhMQ2jkmTJtGrVy/c3NywtLRM/PJLauTIkVy7dg0pJQ0aNMDd3Z2BAwfSvn17Nm7ciJeXV4rtFm9SpkwZ6tWrx4MHD1i4cGGypxqACRMmMHToUNzc3JBSUqxYscQ2DV1q165N9+7dCQwMpGvXrlSpUgUAMzMzvLy8yJcvX2L7g5ubGyYmJri7u+Pj44ONjU2yczVu3JiAgIDXPn9XV1emTJlC48aNSUhIwNTUlJ9++omiRYsmayMZM2YMHTt2ZNmyZTg7O7Nx40YATp8+zcKFC1m6dCnGxsb88MMPNGjQACkllStX5pNPPnnrzzFVUkqDvYBmaNo+rgPjtdsmA620P38HXETTJnIQKKvdboGmTeQS8DfgkeScM9AknCvAUH3iqFy5skxPf19/JIuN2Sm/2OCXrudVsrdLly5ldgg5Ss+ePeXGjRsz5Frx8fHS3d1dXr16NUOuZwi6fj+B01KP71iD/jstpdwN7H5l21dJfh6LptfVq8dFo+m5peucI4GR6Rvp26nuYscgr5LMOxBI3dIOtHIvlJnhKIqSiS5dukSLFi1o27YtpUqVyuxwMoWql3lHgxuU4q/AR4zfch5Pp3w42VpmdkiKkmGqV6/Oixcvkm375ZdfqFixYiZF9LqVK1e+03H79u1j9OjRybYVL16cLVu26Czv6urKjRs33ula2YWQOnpUZDdVqlSRL/ufp6fboc9pNvcIZQta4dvvA0yM1YwzSsoCAgJS7FaqKJlN1++nEOIfKWWV1I5V33xp4GxnyZQ2FTj9bzjzDwamfoCiKEo2pBJJGrXxLExbz8LM/eMap2+FZXY4iqIoGU4lknQwuXV5CtvkYoivHxFRsZkdjqIoSoZSiSQdWFmYMqezJ/cjoxm/5bzOkbyKoijZlUok6aSSsw3DGpZip/89Np+5k9nhKIpOWWkaebUeSdrosx7JwYMH8fDwSHxZWFiwdevWdI9FJZJ09OmHJalW3Javtl3g1qNnmR2OorwmKyWS7C4rrEfi5eWFn58ffn5+HDhwAEtLy8QJLtOTGkeSjoyNBLM7eeA95wiDfc+yaUBNzExUrlZ02DMG7p9P33MWrAjeuhc3ekmtR5Kz1iNJatOmTXh7e2Npmf5j3lQiSWeF8uViWruKfLrmDLN+v8ropmUzOyRFSaTWI8lZ65Ek5evry/Dhw99Y5l2pRGIA3hUd6VLNiYWHrlOnpD01S+r+w1NysFSeHDKCWo8k+69H8tK9e/c4f/584szE6U0lEgOZ0MKVEzfDGLbBj71D6mKT2yyzQ1KUZKRajyTbr0fy0oYNG2jbti2mpqZvPOe7UhX4BmJpZsLczp6EPYth9GZ/1SVYyRLUeiQ5az2Sl9atW0eXLl3eeG9poZ5IDKhCYWtGNy3LlF0BrD15m4+rF83skJQcTq1HkrPWIwFNO1FQUBD16tV7689PX2rSRgNLSJD0XHGSU7fC2DGoNqUKWGVKHErmU5M2ZiwfHx9atGhBhw4dDH6thIQEKlWqxMaNG9/bqeTVpI1ZmJGR4MeO7uQ2M2Gwrx/RsW//eK4oStZ16dIlSpYsSYMGDd7bJJJWqmorA+S3smDGR270Xnma7/de4auW6b9msqJkJLUeyX/UeiQqkWSY+mUL4FOzGMuP3qROaXu8yqTcw0JRsroTJ05kdggG06RJE4N1k82uVNVWBhrjXZYyBawYufEcIU9epH6AoijKe8CgiUQI0VQIcUUIESiEGKNjv48QIkQI4ad99U2yb7oQ4oL21SnJ9gZCiDPa8n8JIUoa8h7Sk4WpMXO7ePIkOo4RG8+RkJD9OzooipL9GSyRCCGMgZ8Ab8AV6CKE0NU4sF5K6aF9LdUe2xyoBHgA1YGRQoiXHc8XAB9LKT2AtcCXhroHQyhT0Iovm5fj0NUQVh67ldnhKIqipJkhn0iqAYFSyhtSyhjAF0h5xExyrsAhKWWclPIZcA5oqt0ngZdJxRq4m44xZ4huHxSlYbn8TNtzmUt3IzM7HCUHyUqz/6pp5NNGn2nkAUaNGkX58uUpV64cgwcPNsjgaEMmksJAUJL3wdptr2ovhPAXQmwSQjhpt50DvIUQlkIIe8ALeLmvL7BbCBEMdAcyf9KitySE4PsO7uSzNGWw71miYlSXYCVjZKVEkt1lhWnkjx07xtGjR/H39+fChQucOnWKQ4cOpfnarzJkInl90h3N00RSO4BiUko34HdgFYCUcj+wGzgGrAOOAy8/+WFAMyllEWAFMFPnxYXoJ4Q4LYQ4HRISktZ7SXe2uc2Y2dGDwIdPmbLrUmaHo+QQSaeRHzlyJDNmzKBq1aq4ubkxceJEQDO1evPmzXF3d6dChQqsX7+euXPnJk4j7+XlleL59+7dS6VKlXB3d6dBgwaA5j/nNm3a4ObmxgcffIC/v/9rx23cuJEKFSrg7u5O3bp1Ac0XcZ06dahUqRKVKlXi2LFjgOaJol69enTs2JHSpUszZswY1qxZQ7Vq1ahYsSLXr18HNAMSBwwYQJ06dShdurTOEevPnj2jd+/eVK1aFU9PT7Zt2/bGz+/lNPJlypRJnPJ9woQJzJkzJ7HM+PHjmTt3LmPGjOHIkSN4eHgwa9YsVq5cyUcffUTLli0TJ5nU9fmDZhr5atWq4eHhQf/+/XVOD7Nt2zZ69uwJaKaR17VglRCC6OhoYmJiePHiBbGxsRQoUOCN9/guDNn9N5j/niIAivBKNZSUMjTJ2yXA9CT7pgJTAYQQa4FrQggHwF1K+bLv4Xpgr66LSykXA4tBM7I9TXdiILVL2dO/rguLDt+gbmkHmpQvmNkhKRlk+snpXA67nHrBt1DWtiyjq41+Yxk1jXzOmka+Ro0aeHl54ejoiJSSQYMGGWR2BUMmklNAKSFEceAO0BnomrSAEMJRSnlP+7YVEKDdbgzkk1KGCiHcADdgv7actRCitJTyKtDo5THvqy8al+Ho9UeM3uyPe5F8FLS2SP0gRUkHahr57D+NfGBgIAEBAQQHByde//Dhw4lPfenFYIlEShknhBgE7AOMgeVSyotCiMnAaSnldmCwEKIVmmqrMMBHe7gpcEQ7RXMk0E1KGQcghPgE2CyESADCgZTnbH4PmJkYMbezJ83n/sWw9X6s7lsdYyNdtYJKdpLak0NGUNPIZ/9p5Lds2cIHH3xAnjx5APD29k5M+OnJoONIpJS7pZSlpZQltFVVSCm/0iYRpJRjpZTlpZTuUkovKeVl7fZoKaWr9vWBlNIvyTm3SCkrao/5UEr53s9N4OKQh69blef4jVAWH37vb0fJwtQ08jlrGnlnZ2cOHTpEXFwcsbGxHDp06L2r2lLewkdVinDoagg/7r9CzRJ2uDvly+yQlGxITSOfs6aR79ChAwcOHKBixYoIIWjatCktW7Z8688xNWoa+Swk4nks3nMOY2ZixM7BdchjrvJ8dqKmkc9Yahr5t6Omkc8mrC1Nmd3Zk9thz5m0/WJmh6Moih7UNPKqaivLqVbclkFeJZl7IJC6pR1o5V4os0NSlNeoaeT/o6aRV4kkSxrcoBR/BT5i/JbzeDrlw8nWMrNDUpRk1DTySlKqaisLMjE2Yk5nT6SEYev9iItPyOyQFEVRUqQSSRblZGvJ1LYVOP1vOPMPBqZ+gKIoSiZRiSQLa+1RmHaehZn7xzVO3wrL7HAURVF0Uokki/u6dXmK2FgyxNePiKjYzA5HURTlNSqRZHFWFqbM6ezB/choxm85b5C1BJScIytNI6/WI0kbfdcjGT16NBUqVEicydkQVCJ5g+i4aGITMv8pwNPZhuGNSrPT/x6bz9zJ7HCU91hWSiTZXVZYj2TXrl2cOXMGPz8/Tpw4wYwZM4iMTP/F9FQiSYGUkq+OfkX/3/oTHq0702ekAfVK8IGLLV9tu8DNR88yOxzlPaXWI0kuu69HcunSJerVq4eJiQm5c+fG3d2dvXt1rryRJmocSQqEELQ4DYcvn6ZrZGfmNJxHaZvSmRaPsZFgVicPms4+whDfs2waUBMzE/V/wPvq/rff8iIgfdcjMS9XloLjxr2xjFqPJGetR+Lu7s7XX3/N8OHDef78OQcPHsTV1fWNvyPvQiWSFEgpKRluhsORWFzv3OfT8K6MazqdBs4NMi0mR+tcTG9fkQGrzzDzt6uM8S6babEo7z+1Hkn2X4+kcePGnDp1ipo1a+Lg4ECNGjUwMUn/r32VSFIghMDxm2+w9PCAb77hu6Uv+OHBEAJbfU4/t346113ICE0rONKlmjOLDl+nTil7apXU/d+hkrWl9uSQEdR6JNl/PRLQVLWNHz8egK5duxpkPjBVN/IGQgjydehA8Q0bsLEpxMS18QQtmMuIg1/wPDbzGh0ntCiHi31uhm/wI+xZTKbFobx/1HokOWs9kvj4eEJDNSua+/v74+/vn/j0lp7UE4keLMqUwWXzZu5N+IqP9+zhTNBePn14i2ktfsIxj2OGx2NpZsKczp60+/kYozf7s7h75Ux7QlLeL2o9kpy1HklsbGxi1WTevHlZvXq1Qaq21Hokb0FKSfi6ddz/7jvCc8Wz9KN8fN5jPpUKVEqHKN/e0iM3mLIrgCltKtDtg6KZEoOiP7UeScZS65G8HbUeSQYRQmDbtSvF1/limyc/X6wIZ/M3Pfn16uZMiad3reLULe3ANzsvce1Byo/QiqIYjlqPxMBPJEKIpsAcwBhYKqWc9sp+H2AG8HKU3Xwp5VLtvulAc+32b6SU67XbjwBW2u35gZNSyjZvisMQKyTGR0Zye8woog8c4lQpwcNhHRn64ZeYGGVsbeHDJ9F4zz6Cg5U5Wz+rhYWpcYZeX9FfdnoieR/WI3lXb7seSXaRlicSgyUSIYQxcBVoBAQDp4AuUspLScr4AFWklINeObY5MBTwBsyBQ0B9KWXkK+U2A9uklP97UyyGWmpXSsmjVSt5OGMGj/JI9n/ixqjui7E2t073a73JwcsP6bXyFL1qFWNiy/IZem1Ff9kpkSjZT1at2qoGBEopb0gpYwBf4PVuBbq5AoeklHFSymfAOaBp0gJCCCugPvD6cM4MIoTAwacXLmvWYmOWj06z/Jn/ZXMCwzN22nevsvnxqVmMFUdvcfDy64OSlKwjJ7RJKu+ftP5eGjKRFAaCkrwP1m57VXshhL8QYpMQwkm77RzgLYSwFELYA16A0yvHtQX+ePUpJTPk8vCg/I49GFWvRLsdoRzt045Dl/dkaAxjvMtStqAVIzedI+TJi9QPUDKchYUFoaGhKpkoWYqUktDQ0Nd6tL0NQ1ZtfQQ0kVL21b7vDlSTUn6epIwd8FRK+UIIMQDoKKWsr903HvgICAEeomkLmZPk2D1o2l10tnQLIfoB/QCcnZ0r6+qHnaqLWyE+Ftw+0qu4TEjg34VzeDZ/MQ+s4cHY7nRuOTbDuuZeffCElvP+4gMXO1b4VMXISHUJzkpiY2MJDg4mOjo6s0NRlGQsLCwoUqQIpqamybZnhTaSGsAkKWUT7fuxAFJKncM1tW0qYVLK1xoYhBBrgdVSyt3a93Zo2l8KSylT/at8pzYSKWFtR7i2Hyr1BO/pYJpLr0PDTx7j+uDPMH0SzamubnQdtQJL04xZd/2Xv/9lwtYLTGjhSp/axTPkmoqiZE9ZoY3kFFBKCFFcCGEGdAa2Jy0ghEg6mq8VEKDdbqxNFggh3AA3YH+Ssh8BO/VJIu9MCOi8DmoPgzOrYGlDeKRf24dNtZq47/qdZxWKUusXf3b0bMC9kJsGCzWpbtWdaViuANP3XObi3YjUD1AURUkjgyUSKWUcMAjYhyZBbJBSXhRCTBZCtNIWGyyEuCiEOAcMBny0202BI0KIS8BioJv2fC91BtYZKvZExibQcBJ8vAki78LienB+k16HmtrZUWPdbp76tKTC2ccEtG3JuRM7DBouaDoAfN/BjXyWpgxed5aomLefVkJRFOVtqJHt+ooIhk29IegEVO4FTb/Tu6rr2u+/EjbmK8yj4wn7vAP1+3+Ttlj0cDTwEd2WnaBLNWe+bfv+9+1XFCXjZYWqrezFugj47IJaQ+GfFbC0EYRe1+vQUg3bUWrbdh4Uy4vjrE3s69eKmGeGHYleq6Q9/eq6sPbEbfZeuG/QaymKkrOpRPI2jE2h0dfQdSNEBsOiunBBv+lRbAu7UH/LYa60cqPI4WucaPEhoVdeXykuPX3RqAwVC1sz5ld/7kVEGfRaiqLkXCqRvIvSjWHAX5DfVVPdtXMYxKbe7m9qak6b79cTNKknZhHPuf1RZ65tWG6wMM1MjJjT2YOYuASGrz9HfEL2r8ZUFCXjqUTyrqyLQK/dUHMwnF4OyxrqXdXVtPMYTFfNJaigCXFfzeDMiAEkvDDMIEIXhzxMalWe4zdCWXRYv/gURVHehkokaWFsCo2/gS7rNY3xi+rBhV/1OrRSxUZU2bCLIx/ak2vnIc60acyLW7cMEuZHlYvQ3M2Rmfuv4hf02CDXUBQl51KJJD2UaQr9j0D+crCpF+z6Qq+qrkL5nOg2bx/7BlZG3nvI5bYtCd2zK93DE0LwbZuKFMhrwRDfszx9EZf6QYqiKHpSiSS95HPSVnV9DqeWwjL9enVZmloy5PNfuPhDb/7NF8fDYSO4+fWXyJj0XULX2tKUWZ08CAp7zsRtF9P13Iqi5GwqkaQnY1NoPAW6+MLj25qqroupr2EghMCn4UhyL53NvuqmRK/bTECn9sQE30n12LdRrbgtg+qXYvOZYLb5pe+5FUXJuVQiMYQy3jDgCDiUgY0+sGsExKXemN6gRBOazd7E8q4ORN0I5Grrljw5cCBdQxtcvySVnPPx5ZYLBIU9T9dzK4qSM6lEYij5nKHXHqgxCE4t0VR1hd1I9bDSNqUZN2oba0Z58q9VFMEDP+P+998jY2PTJSwTYyPmdPYEYOh6P+LiE9LlvIqi5FwqkRiSiRk0mQqd10L4LW1VV+rrcNlY2DC9yyrOTe7MPk9B+PIV3Ojejdj76TNC3cnWkiltK/DPv+HMO5Cxi3ApipL9qESSEco21/Tqsi8FG3vC7pGpVnWZGpkyru5ECk2axLzWpjy5dJ7A1q15euRIuoTU2qMw7SoVZt6Ba5y6FZYu51QUJWdKNZEIIUoIIcy1P38ohBgshMhn+NCyGZui0Guvpqrr5GJY1hjCUp9avmOZjvQatoyp/fIRZP6MoE/68XD2bGRc2rvwTm5dgSI2lgz19SMiKn2qzhRFyXn0eSLZDMQLIUoCy4DiwFqDRpVdJavquqmZq+vStlQPq1qwKrN9NrJ8cCkOuBsRunARt3v3IfZh2tZnz2NuwtwunjyIjGbclvNqCVhFUd6JPokkQbsWSFtgtpRyGOCYyjHKm5RtDv0Pg11J2NAD9oxOtaqrcJ76vqR5AAAgAElEQVTCLG+9hsABjZnfwohIv3+42bYdz/7+O02heDjlY3jj0uzyv8emf4LTdC5FUXImfRJJrBCiC9AT2KndZvqG8oo+bIpB733wwUA4sRCWN9U0yL+BpaklP374I+W7fcaoHnDf5Bm3e/Um5KefkPHvvoBV/7olqOFix8TtF7n56Nk7n0dRlJxJn0TSC6gBTJVS3hRCFAdWGzasHMLETLNAVqc1mlHwC+tCwJtXUTQSRgz0GMiwDjMZ72PCCTcLHs2bT9An/YgLDX2nMIyNBDM7uWNmYsQQ37PExKkuwYqi6C/VRCKlvCSlHCylXCeEsAGspJTTMiC2nKNcCxhwGOxcYH032DMG4t48RUrjYo1Z0voX1nawY2lzM56cPsnNtu14/o4rQTpa52JaOzf8gyP4bk/AO51DUZScSZ9eW38KIfIKIWyBc8AKIcRMw4eWw7ys6qr+KZxYAMubQPi/bzykrG1Z1rXw5VFDD0Z3k0QYveDfnj48WrIEmfD2TxVNKxSkV61irDh6iyWHUx88qSiKAvpVbVlLKSOBdsAKKWVloKFhw8qhTMzBexp0/EVT1bWoDgTsfOMhdrnsWNp4KVVqd2Bg16dcc7cj5MeZBH36KXHh4W8dwpfNXWnu5sjU3QH8ekY1viuKkjp9EomJEMIR6Mh/je16EUI0FUJcEUIECiHG6NjvI4QIEUL4aV99k+ybLoS4oH11SrJdCCGmCiGuCiEChBCD3yam94JrK+h/CGxdYP3HsHfcG6u6TI1NmVhjIsPqjuerJo/Z2jo/z44d52a79kT5+b3VpY2NBDM7ulOzhB2jNvlz8ErauhgripL96ZNIJgP7gOtSylNCCBfgWmoHCSGMgZ8Ab8AV6CKEcNVRdL2U0kP7Wqo9tjlQCfAAqgMjhRB5teV9ACegrJSyHOCrxz28f2yLa6q6qvWHv3+CFU3fWNUlhKBL2S4saryYXZ7xfOOTixjiuNWtO6ErV77VGBFzE2MWda9MmYJWDFx9hrO33/7JRlGUnEOfxvaNUko3KeWn2vc3pJTt9Th3NSBQWz4GzRd+az3jcgUOSSnjpJTP0LTNNNXu+xSYLKVM0MaTff9lNjGHZt9Dx//Bo2uaqq7Lb174qrpjddY1X0dUyUJ80iWCx5VL8HDadII//5z4yEi9L21lYcrKXtXIn9ec3itPEfjwaVrvRlGUbEqfxvYiQogtQoiHQogHQojNQogiepy7MBCU5H2wdtur2gsh/IUQm4QQTtpt5wBvIYSlEMIe8ELzFAJQAugkhDgthNgjhCiVQtz9tGVOh4SE6BFuFubaWlPVZVMcfLvCvvFvrOpysnJidbPVVC5Zl35egZzp7MHTPw9pqrrOX9D7sg5W5vyvdzWMjYzosewE9yKi0uNuFEXJZvSp2loBbAcKoUkEO7TbUiN0bHu1fmUHUExK6Qb8DqwCkFLuB3YDx4B1wHHg5eRS5kC0lLIKsARYruviUsrFUsoqUsoqDg4OeoSbxdm6QJ/9UK0fHJ8PK7w1i2elILdpbuZ4zeETt35MK36BVZ+VIj4uhn+7diVszRq9q7qK2uVmZa+qREbH0XP5SSKeqzm5FEVJTp9E4iClXKGtZoqTUq4E9PlmDua/pwiAIsDdpAWklKFSypdzgywBKifZN1XbbtIITVJ62S4TjGb+L4AtgJsesWQPJubQbAZ8tAoeXYWFdeDKnhSLGwkjBlcazIy6M/jDKohRvY2RVSry4Jsp3Bk+nPin+lVXVShszeLulbn16Dl9Vp0iOvbdR9EripL96JNIHgkhugkhjLWvboA+Q6hPAaWEEMWFEGZAZzRPNom0vcFeagUEaLcbCyHstD+7oUkW+7XltgL1tT/XA67qEUv2Ur4N9PtTs3jWus6w/0uIT/lJoWnxpqzyXsWzXEb0rH+F8N4teLL/N26170D05ct6XbJmSXtmdfLgn9vhDFp7Vi2IpShKIn0SSW80XX/vA/eADmimTXkj7USPg9D0+AoANkgpLwohJgshWmmLDRZCXBRCnAMGo+mRBZq5vI4IIS4Bi4Fu2vMBTEPTrnIe+A5I7DKco9iVgD6/QdVP4Ng8bVVXUIrFXe1c8W3hSxm7cvQvsJfTX7YkISqKW506E75hg15VXc3dHJncqjy/BzxQswUripJIvMuXgRBiqJRytgHiMYgqVarI0+84dch74eIW2PY5GBlD20VQpmmKRWPiY5jy9xS2BG6hWb5aDNgaTfTxE+Rt1RLHiRMxyp071cvN/O0qc/+4xmdeJRjZpGx63omiKFmIEOIfbXv0G73rConD3/E4xRDKt9X06srnBOs6vbGqy8zYjK9rfs2YamPYG3GcL1o9xqx/DyJ37ORmx068uJbqECGGNSxFl2rO/HTwOiuOpr44l6Io2du7JhJdPbKUzGRXAvr8DlX6aKu6mkGE7ilOhBB8XO5jFjRcwP3oh/R03MXzH0YRHxHBzY6dCPvfLyTEpNy9WAjBlDYVaFK+AJN3XmL7ubspllUUJft710SiKsezIlMLaDETOiyHhwGwsDZc3Zdi8ZqFarKu+TrsLOzoEzob/+97YOnpwYNvv+V64yaEr1uXYkIxNhLM6exJ1WK2fLHBj7+uPTLUXSmKksWlmEiEEE+EEJE6Xk/QjClRsqoK7TVVXdZFYG1H+O2rFKu6iuYtyupmq6lVuBZfX5vLyr5FKbR0MaaOjtz/ejLXmzQl3NcXqSOhWJgas6RHFUo45KH/L6fxD35s6DtTFCULeqfG9vdNtm9sT0lsNOwbC6eXg1N1zZOKte5JCeIT4pl7di7LLyynUv5K/FDvB3KdvcqjefOJ8vPDpJAj9v0HkK9tG4SZWbJjH0RG037BMaJi4tn0aU2K26feYK8oStZn6MZ25X1gagEtZkH7ZfDgomYA49X9OosaGxkzrPIwptWZxsXQi3TZ1YV/y+aj6Lq1OC1diqlDfu5PnMj1pt6a7sJJnlAK5LXgf72rIYEey0/wMDI6g25QUZSsQCWSnKBiB+h3CPIWhrUfwW8TU6zqau7SnP95/w8jYUTPPT3ZeWMneWrXoqjvOpyWLMbYwZ77X03kunczwjduRMZqzuPikIcVPlUJfRpDzxWniIxWU6koSk6hqrZyktgo2DsG/lkJTh9oq7p0zaMJYdFhjDg0glP3T9HdtTvDKw/HxMgEKSXPjhwhZN58os+fx7RwYew/HYB169YIU1MOXw2hz6pTVC5qw8pe1bAwNc7Ye1QUJd3oW7WlEklO5L8Rdg7VzN3VdjGU0r3gZWxCLD+e/pE1AWuoXrA6M+rNwMbCBkCTUA4f1iSUCxcwdXLCfsAArFu1ZPvFhwzx9aNp+YL89HEljI1Ub3FFeR+lWyLR9tJ6tVAEcBr4QkqZ5Rf3VolEh0fXYKMPPLgAtYeD13gwNtFZdGvgVr45/g0Olg7M8ZpDGdsyifuklDz9808ezf+J6IsXMXV2xn7AAH61q8jkPVfoWt2ZqW0qIIRKJoryvknPRPI1mll716IZiNgZKAhcAT6VUn6Y5mgNTCWSFCSt6nKuoanqyqu7Z/f5kPMM/XMokS8i+abWNzQtnnwaFiklTw/+yaP584m+dAlTZ2dO1m7Nl0+LMLhxWYY2LJ0BN6QoSnpKz0RyQkpZ/ZVtf0spPxBCnJNSuqcxVoNTiSQV/htgx1BNL692i6Gk7qquR1GPGHZwGH4hfvSp0IfPPT/H2Ch5G4gmoRwkZP58XlwKIMKuIIuLelH/s250q+WSEXejKEo6Sc/uvwlCiI5CCCPtq2OSfdm/gSUncOuomZY+TwFY3R5+/xri414rZp/LnuVNltOhdAeWXVjGZwc+I+JFRLIyQgis6ten+ObNFJk/Dwd7a0aeWUeBIT4c/ul/yHi1lomiZDf6PJG4AHOAGtpNx4FhwB2gspTyL4NGmA7UE4meYp7D3tFw5n/aXl3LUhzAuOHKBr47+R2FchdijtccStqU1FlOJiQQuu83/Kf+gOOjYOILO+E09HPyNmuGMFY9uhQlK1O9tpJQieQtnd8EO4aAsSm0/hnKNtNZ7MyDMwz/czhRcVF8W+dbGjg3SPGU4U+jmTJ2IY1ObqdoxD3MihfHfuBA8jbzVglFUbKodKvaEkIUEUJsEUI8FEI8EEJsFkLo/jdVyR4qdoD+hzUrMPp2gT1jIO7Fa8UqFaiEbwtfXKxdGHpwKD/5/USC1L1yok0eC0ZM6c83rccyt24fYoURd0eO5Ear1kTs2qWqvBTlPaZPG8kKNEvkFgIKAzu025Ts7OUKjNUHwIkFsKwxhL3e07tg7oKs9F5JqxKtWHhuIUMODuFpjO614B2tc7Gq7wccL+LGpx8OI++07xFGgrtfjOBG69ZE7t6NTFBL+CrK+0afROIgpVwhpYzTvlYCDgaOS8kKTMzBezp0WgPht2BhXU211yvMjc2ZUmsKY6qN4UjwEbru7sqtiFs6T1kyvxXLelbl3pMYBgTZ4LBhM4VnzQTgzvAvuNm6NZF79qiEoijvEX0SySMhRDchhLH21Q0INXRgShZSrgUM+AsKuMLmPrB9sKZhPomXi2UtabyEx9GP6bKrC4eDD+s8XeWiNiz4uDKX7kUyYM0ZzBs1xmXbNgrP/BGZILkzbDg3W7chcu8+lVAU5T2gTyLpDXQE7gP3gA5AL31OLoRoKoS4IoQIFEKM0bHfRwgRIoTw0776Jtk3XQhxQfvqlGT7SiHEzSTHeOgTi5JG+ZzAZxfUHgZnVsGS+vDw8mvFqhasim8LX5ysnBj0xyCW+C9BV4cOr7L5+b69G0cDQ/liwzmkMCJvs2a4bN9GoR9+QMbHc2foUG62aUvkvv0qoShKFvZOvbaEEEOllLNTKWMMXAUaAcHAKaCLlPJSkjI+QBUp5aBXjm0ODAW8AXPgEFBfShkphFgJ7JRSvl7HkgLVayudBf4Ov/aHmGfQbAZ4doNXpkCJioti4rGJ7Lm5h0ZFGzGl1hQsTS1fO9WiQ9f5bs9lfGoWY2JL18SpVGR8PJG79/Do55+JuXkT8zJlsP9sIFYNGyKM1KTVipIRDL0eyXA9ylQDAqWUN6SUMYAv0FrP87sCh7RtMs+Ac0DTVI5RMkrJhvDpUXCqCtsHwa/94MWTZEVymeRiep3pjKgygj9u/8HHuz8mKDLotVP1q+tC39rFWXnsFj//eT1xuzA2xrplC1x27qDQjO+RL15wZ/AQbrZrT+Rvv+l8ylEUJXO8ayLRZwa+wkDSb45g7bZXtRdC+AshNgkhnLTbzgHeQghLIYQ94AU4JTlmqvaYWUII83e5ASWNrApC962ayR4vbIJF9eCef7IiQgh6lu/JggYLePj8IZ13debY3WOvlRnXrBxtPQszY98V1p+6nXy/sTHWLVtqEsr0acioKO58Ppib7drz5I8/VEJRlCzgXROJPn+9upLNq8ftAIpJKd2A34FVAFLK/cBu4BiwDs1o+pdzdowFygJVAVtgtM6LC9FPCHFaCHE6JCREj3CVt2ZkDPVGQc+dEPscljaAk0vglS/3moVr4tvcl/yW+fn0909ZeWFlsgRgZCT4voMb9Uo7MPbX8/x26cFrlxImJli3bo3Lrp04TvuOhOfPCP5sEDfbt+fJgQMqoShKJkqxjSSF6eNBkyBySSl1zzn+3/E1gElSyiba92MBpJTfpVDeGAiTUlrr2LcWWC2l3P3K9g+BEVLKFm+KRbWRZIBnobD1U7i2D8q2gNbzIZdNsiLPY5/z5dEv+e3f32hWvBmTak4il0mu/07xIo6uS09w+V4kq/tWp2ox2xQvJ+PiiNixk0cLFhB7+zYWrq7YDxpEHq8P1ZT1ipJO0txGIqW0klLm1fGySi2JaJ0CSgkhigshzNBMP7/9lSAdk7xtBQRotxsLIey0P7sBbsD+pMcIzbdFG+CCHrEohpbbDrr4QuOpcHWvZsxJ0KlkRSxNLfmx3o8M9hzMnpt76LmnJ3ef3v3vFOYmrPCpSmGbXPRZeYor95+8epVEwsSEfG3bUGLXThynTiX+yROCBw7k1kcdefLnn+oJRVEykEHn2hJCNANmA8bAcinlVCHEZOC0lHK7EOI7NAkkDghDs77JZSGEBXBGe5pIYICU0k97zgNoBkQKwE+7T/dQai31RJLBgv+BTb0g8g7UnwA1B8MrPa0OBx9m9OHRmBqZ8uOHP1K1YNX/Dg9/TvsFmraUzZ/WpIjN6729XiVjY4nYvp1HCxYSGxyMRcWKOAz6jNx166onFEV5R2rSxiRUIskEUY9hx2C4tE3Ty6vtIshtn6zIzYibDDk4hNuRtxlVdRRdynZJ/NK/fD+SjguPY29lzqYBNbHNbabXZWVsLBHbtmkSyp07WLi5aRJKnToqoSjKW1KJJAmVSDKJlHB6Oewdq2kvab8UitdJVuRJzBPGHRnHn8F/0qZkG7784EvMjTUd8U7eDKP7shOUdczLuk+qY2mmT42q9tIxMTzeto3QBQuJvXsXC3c3HAZ9Tu7atVRCURQ9qUSShEokmez+ec368GE3oO4oTU+vJCsrJsgEFpxbwMJzC6loX5FZH86iQO4CAOy/eJ8Bq/+hTikHlvasgqnx23U0lDExPN6ylUeLFhJ39x65PDywHzSI3LVqqoSiKKlQiSQJlUiygBdPYfcIOLcOitaG9kteWx/+j3//YNxf48hlkotZXrPwzO8JgO/J24z59TxtPQvz40fuGBm9fQKQMTE8/nULjxYtIu7ePXJ5emI/6DNy11QJRVFSYuiR7YrydszzQNuF0GYB3D0DC2vDtd+SFWlQtAFrmq0ht2lueu/rzcarGwHoXM2ZEY1Ls+XsHabtfX1+L30IMzNsOneixL69FJw0kdh79wjq05d/P+7Gs2PHVC8vRUkDlUiUjOXRFfodAitHWNMB9k+A+NjE3SVtSrK2+VqqO1Zn8vHJTD4+mdj4WD7zKknPGkVZfPgGiw9ff8MF3szIzAybzp0psX8fBb6aQOydO9zu3Yegvp8Q8++/6XGHipLjqKotJXPERsG+cZrG+CJVof0ysCmauDs+IZ65Z+ey/MJyPPN7MvPDmdia2/G571l2+d/jx4/caV857Qt1Jrx4wWNfX0LmzkPGxmLXvx92n3yCkZl+vcQUJTtTVVtK1maaC1rMgo9WQsgVWFQHLv03XtXYyJhhlYcxo+4MAkID6LSzExdDLzCzozu1StoxarM/By8/THMYRubm2Pbsicvu3Vg1bMCjefO52ao1z44fT/O5FSWnUIlEyVzl22rWh7ctARu6w64REBuduLtp8aasbrYaUyNTfPb6sOfWDhZ2q0w5RysGrjnDmdvh6RKGaYH8FJ45E6elS5EJCdzu1Zs7I0YSp+ZpU5RUqUSiZD7b4tB7H9QYBKeWwLKG8CgwcXcZ2zKsa74Oz/yeTDg6gZ/8f2RJj0rkz2tO75WnCHyY8lQqbytP7Vq4bN+G/cCBPNm3j+vNmhO+bh0yPj7drqEo2Y1KJErWYGIGTaZCl/UQEQyL6sK59Ym7bSxsWNhoId1du7MmYA3j//6c+R+XxsTIiB7LTnIvIirdQjGysMBh8OcU374Niwrluf/1ZG517kLUxYvpdg1FyU5UIlGyljJNYcBRcHSHLf1g62ealRgBEyMTRlUdxbe1v+Xcw3OMONabSR2siYyOo8eykzx+HpOuoZgXL47z8uUUmjGD2Lt3ufVRR+5P/Zb4p2+c2k1RchyVSJSsx7ow9NwBdUeC3xpY7AUP/nsaaFmiJf/z/h/xMp6vTw+kr/dj/g19Tt9Vp4mKSd8qKCEE1i1bUGLPbmw6dyJ89WpueDcjcs8eNfZEUbRUIlGyJmMTqP8l9NgKUeGwpD6cXpG4aFZ5+/L4tvDF1c6VZVem0Lj2Kf65Hcrn684QF5+Q/uHkzUvBr76i2HpfjB3suTNsOEGf9CPm9u3UD1aUbE4lEiVrc/lQsz68cw3YORQ29YboSADsc9mztPFSOpfpzKGHG6lQaSO/X7nFuC3nDfa0kMvNjeIbNlBg3Diizp7lRouWhPz8Mwkx6VutpijvE5VIlKwvT37o9is0mKiZln5RHbijWa7G1NiU8R+MZ1KNSdyJvkAh10VsOn+aGfuuGCwcYWKCbY/uuOzeTZ4G9Xk0d55m7MnffxvsmoqSlalEorwfjIygznDotRvi42BZYzj+c2JVV/vS7VneZDkWZgnkdVnA4n+2svyvmwYNybRAforMmoXTkiWasSc+vbgzchRxjx4Z9LqKktWoRKK8X5w/gAFHoFQj2DcWfLvC8zAAPPJ74NvCl/L2ZchVZA3TTsxiq1+wwUPKU6e2duzJpzzZu5fr3s3U2BMlR1GJRHn/WNpC57XQdDoE/q6ZSfhfzZQm+S3zs6LpClq5tMHc/iDjjg5jf4Bhn0zg5diTwRTftg2L8mrsiZKzqESivJ+EgA8GQJ/9YGwGK5vD4R8gIQEzYzOm1J7M8EpjMM59leF/9WHvlXMZEpa5S3GcV6ixJ0rOYtBEIoRoKoS4IoQIFEKM0bHfRwgRIoTw0776Jtk3XQhxQfvqpOPYeUII9deZ0xXy1MzVVb4NHPgGVreFJw8QQtCr4sfMqrsAI+MoRh7rg++FPRkSUopjT/buVWNPlGzJYIlECGEM/AR4A65AFyGEq46i66WUHtrXUu2xzYFKgAdQHRgphMib5NxVgHyGil15z1jk1UxD33Iu3P5bU9V1/SAADV1qsrTRGkSsA1P/GcWPJ+eTINN/nIkur409GTqMoH791dgTJdsx5BNJNSBQSnlDShkD+AKt9TzWFTgkpYyTUj4DzgFNITFBzQBGGSBm5X0lBFTuCZ8c1LSh/NIW/vgG4uOo5uTC8iYrSXhSiZUBi/j8j6E8i32WYaH9N/ZkLFFnznCjZSseLVigxp4o2YYhE0lhICjJ+2Dttle1F0L4CyE2CSGctNvOAd5CCEshhD3gBbzcNwjYLqW8Z6jAlfdYAVf45AB4fgxHfoBVLSAimCpFC7Cg8QxiH7bgcPAhuuzqyu3IjHsy0Iw96YHL7l3k8fIiZM5cbrZuo8aeKNmCIROJ0LHt1QriHUAxKaUb8DuwCkBKuR/YDRwD1gHHgTghRCHgI2BeqhcXop8Q4rQQ4nSIWlMiZzHLDa1/gnZL4f55TVXXlT3ULZ2f6Y0G8fx2b4IiHtB5V2eO3jmaoaGZFihAkdmzcFqyGBkXp8aeKNmCIRNJMP89RQAUAe4mLSClDJVSvtC+XQJUTrJvqrbdpBGapHQN8ARKAoFCiFuApRAiEB2klIullFWklFUcHBzS656U94nbR5qGeGsnWNcZ9o6jdQUHvmzQiseBn0GcDQP/GMjyC8szvBE8T506uOzYjv3AT4lUY0+U95zB1mwXQpgAV4EGwB3gFNBVSnkxSRnHl1VUQoi2wGgp5QfadpB8UspQIYQbsBbwkFLGvXKNp1LKPKnFotZsz+HiXsD+CXBykaaXV4flfH8yhp8PBeDmsZeb0cfIa5YXJyunZK8iVkVwsnIiv2V+jITh/ud6ceMm9ydP5vnff2Ph5objpIlYuOrql6IoGUvfNdsNlki0QTQDZgPGwHIp5VQhxGTgtJRyuxDiO6AVEAeEAZ9KKS8LISyAM9rTRAIDpJR+Os6vEomiv4AdsO0zkBLZcg6jL5dgw+kgOnuFkMf6DkFPgrgdeZt7z+4RL/97MjA3NqdwnsKvJRhnK2cK5ymMqbFpmkOTUhK5cycPpk0nPjwcm24f4zB4MMZ5Uv31VhSDyRKJJKtQiURJ9Pi2Zgbh4FMkVPJh4KOP2HctgtbuhShqlxtnW0sc85lhkSuS5wkPuPM0mKAnQZrX0yCCnwQTFfffaoxGwoiClgWTJZikrzxmb5cI4iMieDh7No9912Pi4ECBcWOxatIEIXQ1OSqKYalEkoRKJEoy8bGawYtH55Dg4Mp3uUez854V9yOjSfrnYGZiRBGbXDjbWuJkY4mzrSVFbCzImycaaRJKWMy9/5LME02SCYsOS3YpG3ObFJOMfS77FBNElL8/9yZN4sWlAHLXqUPBCV9i5uxsyE9FUV6jEkkSKpEoOl37Hbb0h9jnULUPMSWaEJynIrcfxxAUHkVQ2HOCwp5zW/t6Ep2siY58lqaJScbJ1hIn21w45JWYmIXzQoRw73lwsiRz79m9ZIMhc5nkSlZllvTlmMcRkwRB+Nq1hMyZi4yLw35Af2z79MHIzCyjPyklh1KJJAmVSJQURd6D3SPg6j5IiAULayjZCEo3hZINNIMbtSKexxIU/l9ieZlkgsOjCA5/Tmz8f39LRgIcrXPhZPvfE00hG1MsckWCSSiRcfcJevpfkgl+Ekx0fHTi8cbCmIK5NVVmpeLsqLnxMnbHr4BzYfJ/9SV2tT/MyE9JyaFUIklCJRIlVS+eaKZVubpXk1SePwJhrJm2vnQTTWKxL60ZQa9DfILkQWR0YoIJCntOUHhU4vuHT14kK5/L1DgxyRSxscTJxgJrq2iMzcOIFSE8iLpDUOR/bTMRLyJwv55An/0JFHwMf7uZ83f7MtgWcnntacbWwla1qSjpQiWSJFQiUd5KQgLcPaNNKns1gxoBbIppEkrpplC0FpjoX8UUFRNPcPhzzRNNaPIkExT2nGcxyceP2Ocx01SXadtmHKzjMcsVjoy7j+3mHRTZdopYM8G2RlZscX1KQpK8YWliqbMbs5OVEwVzF8TEyOT/7d15fFXVvffxz+8MmXPCPIYIyDwro7PVljGg97lWa20vvlq19nG42orWti+ttH2Ecqv24Xq1TrX11g5Pr1YDiGgdQMvogBJmIUAYQyA5mXOG3/PH3iGHECAmOZn4vV+v8+KcvdfZe62Ec77Za++9VjP8kMy5wIIkhgWJaZLifNixArYth93vQ7gSEtLh/K84oTJ4KqQ1/qZXVeV4eeikLjPniMZ5faCokki09nPq8wgXUMx31v8/+u/bQg3OolMAABsSSURBVHH/weR95yYqB6cTkgKOVh0g373abH/JfqqjtWN6+cRHn7Q+ZKZn0jWpKxmJGXRK7ESnxE5kJNU+75TYiYzEDJJ9yU360bUUVSUUUUKRKKFIlOpIlOpw9MSy6nAUjwjDe6fb0dqXYEESw4LENJvqcti9srYLrOQAINB3vHu0Mg16jT5tF1hjhCNRDhZXnjgn4wRMBXsLy8j6eCU3fvQqgaoyXj//Ul4aNg1vWhr9uqSQ1SWZzE5JdM6oICHpOFHvUcqihzlYvp/8knyOVx6nqKqI8nD5afed6E0kIyGDQGIGgYQM0v0ZpPkDpPkDpPgCpHgDJHvTSfKmkyhpJHoC+CWFcMRT+6Uervly1zqvo4TC9SyLCYHqOtsIRZTqcG25UEy5hpjYvzOPzBnFiD6Bsxc2FiSxLEhMXKjCoc+cQNm+HPZ/5CwP9K09rzLgcvDH96/6koJC8hc9hua8SnVGFzZk38zqvmPYe7yCfcfLqQyd/CXbM5BIZucUvCLOX+6RKqqipVRFSwhRRkhLCUspEUqJSjlRTxniLT/1Iaf/8tZIEhpJRSMpp314SSVB0/BJGn5Jw+9JItHrJcHnwe/14PcKfq+HBJ+HBK+7zOcsTzxRxnkkeCXmfU65RK8Hv09OLDtYVMFv/rGD4ooQ355yHj+YOpSM5KbfTNqRWZDEsCAxLaLkMOx8ywmVL96F6lLwJcPAK2qDJdAnbruv2LiRgz97hKottfee+Pv1o6C0yu0uqz0vk3+8AkVJ8HlJ8NZ+2dZ8GZ9Y5n6JJ/hO/mL3eQSVSqq1lGotcR7RUiqiJVSESyiPBCkPBykNBymtLqYkFCRYXUx5+PTD9/s9/hNdavX9W9/zjMSML3XOp6i8ml+v2M4f1+6hc0oCD0wfxnXjM/F4rLurPhYkMSxITIsLV0HeB+7RyhvOHfUAvcbUnrDvcwF4mncMLw2HnXtPnvgNGom0uXtPQpEQxdXFFFcVU1RVRFFV0anPK2ufF1c768LR8Gm3meZPO2PY1PzbLbkbQzoPQUTYtL+Yh17bxMd7i7ggqxPz54xidGZGC/4k2gcLkhgWJKZVqULBttqrwPatBY1CancYPM05Wjn/K5CY3my7DB0+zOFHF1CyfDkJAwbQ6+GHSZ0yudm235JUlfJweW3YVJ4aQjWBU7OuuKqYklDJKdua2GsiD0x8gKFdhhKNKq98sp8Fb2yhsKyaGydlMW/qUDqnto3QbQssSGJYkJg2pfwY7HzbCZUdb0NVMXgToP+ltSfsO/dvll2VrlrFofk/J7RvH4E5s+l5//34unVrlm23deFo2DmqccNmy7EtPL3xaYLVQa4bfB13XnAnnZM6E6wM8fhb2/nD6j2kJ/mYN20o35iYhde6uyxIYlmQmDYrEnKOULa94XSDFe5wlncf5p5XmQGZE8Hb+Hs/opWVHP3tbyl87nk8ycn0+MG9dLr+eqSZu9Xag+KqYp7a+BR/3vpnUvwp3DHuDm4YegM+j4+th4I89Fou63YfY3TfDB65ZiQXZnVu7Sq3KguSGBYkpt0o/KL2KrA9H0I0DMmd3WFbpjnDtiQ37sutatcuDj0yn/K1a0kaO4Zut91G2mWXIW3k/ElL2nl8JwvXL2TNwTUM6jSI+yfez0V9LkJVeX3jAX65dAtHSqq4fkImD0wfRte0xNaucquwIIlhQWLapcri2mFbdqyA8kJ32JaLYoZtGfyl7llRVYI5ORxetIhIwVE8GRkEpk0jY3Y2yePHn1NHKarKu/veZdH6ReSX5nNVv6u4b+J99EvvR2lVmMX/2MHzH+wmJcHLD6cO5abJWfi8587PByxITmJBYtq9aMS5T6XmRsjDm5zlXQbWnlfJurjBw7ZoKETZ6tUU5yyh5O230YoKfH16kzFrFoHs2SQNHRLHxrQtVZEqXtr8Es989gzhaJi5I+dy6+hbSfGnsPNICQ+/nsuHOwsZ3jvA/GtGMrF/l7NvtIOwIIlhQWI6nKJ9taGyeyVEqpxhWwZdVTtsS2rDTqpHy8ooeecdinNyKPvwnxCJkDhkCIHZ2WRkZ+Pv3TvOjWkbjpQf4YmPniBnVw49kntwz/h7yB6YDcAbmw7xiyWbOVBcyb9c0JcHZwyjRyCplWscfxYkMSxITIdWXQa73q8NltJDgDgn6Wu6wHqObFAXWLiwkOAbywnm5FCxcSMAKRMmEJg9m8D0aXgzOv69FhsLNrJg7QI2FW5ibPex/GjSjxjVbRTl1WGefHcnz67cTYLPwz1fHczci/vj78DdXRYkMSxIzDkjGoVDG2tP2B/4xFkeyKwzbMvZ/5qu3ruX4iVLCOYsoXr3bvD7Sbv8cjJmZ5N25ZV4kjruX+RRjfL6F6/zxEdPUFhZyLWDruXfL/x3uiV3Y/fRMh7JyeW9bQUM6ZnGI3NGcdH5XVu7ynHRJoJERKYDvwG8wHOquqDO+puBRcB+d9F/qupz7rqFwCx3+c9V9S/u8ueBCYAA24GbVbX0TPWwIDHnrJJDtSMX73rXmQ3SnwIDrnACJWuyc7e99/RjTqkqlbmbCebkEFy2jHBBAZ60NNKnTiVjdjYpkyYhXm8LNqrllFaX8sznz/DS5pdI9CbyvTHf41vDv4XP4+OtzYeZv2Qz+ccrmD22Dz+eOYzeGe1jtOSGavUgEREvzhf914B8YD1wo6pujilzMzBBVe+s895ZwD3ADCAReB+4SlWDIhJQ1aBb7jHgSN2AqsuCxBggVOkO2+JeBVa0x1nuT3FGL866yJnIK3MiJNU/Oq5GIpSvXeucpF+xgmhZGb7u3QnMmkVgdjZJI0Z0yGHa9wT3sGj9It7Pf5/zAudx/8T7uTzzcipDEZ567wueev8LfB7hrqsG891LB5Dg6xjdXW0hSC4Cfqaq09zXDwKo6qMxZW6m/iCZBySq6i/c188Db6rqX2PKCPBfQJ6qLjxTXSxIjKlH8X7Ytwb2roW9q50rwTQK4oEeI51QqXlkZJ7y9mhlJaXvvUdxzhJKV66EUIiEgQPJmJ1NIDubhH79WqFR8fXB/g9YuG4hecE8Lu17KfMmzmNgxkD2FpYzf8lm3t5ymIHdUvnZnJFcPqTxc9S0FW0hSK4DpqvqLe7rbwOTY0PDDZJHgQKco5d7VXWfiEwFHsY5mkkB1gFPquqv3ff9DpgJbAZmqerpJ1TAgsSYBqkqgfz1tcGSvwFC7mi9gcyTg6XHCPDUdmdFiooIvrmCYE4O5e5nLXncOAKzswnMmIGvS8e5ZDYUDfGnLX/iqY1PURmu5JvDv8ntY28nPSGdd7ce4ZGcXPIKy5k+shc/zR5OZueUlq2gKpQecf4wOJwLk25t9FQGbSFIvg5MqxMkk1T1rpgyXYFSVa0SkduB61X1KnfdT4Cv44TMEWCdqv4m5r1eYDGwXlV/V8/+bwNuA8jKyhq/Z8+euLTTmA4rEobDn9cGy9417hVhQGLA6QLLusg5z9J3PCSkAhA6cIDipUsJ5iyhavt28PlIveRiMrJnk371VXhSWviLNU4KKwpZ/MliXtnxCp2TOnP3BXdz7aBrCUXg+Q92s/gdZ7ibO64cxK2XDyTJH4fzSKFKOLrNCYxDm2rDo/xobZnvrYLeYxq1+bYQJGft2qpT3gscU9VTri8UkZeB/1bVZXWWXwHMU9XsM9XFjkiMaQaqznmVmmDZtxaOuKc8PT7npH1NsPSbAuk9qdy2jWBODsVLlxE+eBBJSSH96qvJmJ1N6sUXI772P3/85sLNLFi3gE+OfMLwLsN5cPKDXNDjAvYXVfCLJZt5Y9MhzuuawsOzR3DVsJ6N24kqBPc7IVETFodz4egO0IhTxpcEPYY7l3r3HO3+OxJSGn802BaCxIfTXXU1zlVZ64FvqmpuTJneqnrQff4vwAOqOsUNlU6qWigiY4CXgXFABDhfVXe650gWAajqfWeqiwWJMXFScRz2ra8Nlv0fOXPaA3QecCJYNHMy5XlBgkuWEnzzTaLBIN4uXQjMmEHG7GySxo5t1yfpVZU3dr/BYx89xuHyw8wYMIMfjP8BvVJ7sWpHAQ+/nsuugjK+OrwHD2WPJKvrGY7KqsvgyFY3MDbVhkdlcW2ZTlnQc1RtWPQc5Yxy4Gneo55WDxK3EjOBJ3Au/31BVX8pIvOBDar6uog8CswBwsAx4PuqulVEkoCP3c0EgdtV9VMR8QCrgADO5b8b3fcEz1QPCxJjWki4Gg5urA2WvaudMcLAGWyy3xSivSdQdiiZ4n9upfT9lWh1Nf6sLDKyneFZEgcOaN02NEF5qJwXNr3Ai7kv4hEP3x31XeaOnIuHBH734W5+848dhKPK7Vecz/++YgBJpfm1Rxc1oXFsF+B+LyekOeejagKj12jnqCOpZW4MbRNB0lZYkBjTSlSdEY33rnavEFsDhTuddd4EIl3HUlLYm+LPiyn/fCeokjRyJBlzZhOYORNf9/Z55dP+0v38esOveWvPW/RN68sPx3yfryb0ILhnI59/9CFJx7cy3LOPVCrcd4hzRFFzdFETHJ3Oa/ZZNL8MC5IYFiTGtCGlBc7RSk2wHPgUoiFCFR6CR88jmJdA5f4S8HhInTKFwOzZpH/tq3jT0lq75mcXjThHFO7RxbqDa1lQmccOH0yqqOSBwuMM8aYQDAzlnaLurCnrTXLmGP7tmhkM6NOjtWt/CguSGBYkxrRhoQrY/3FtsOxbS9XhMor3JBPcl0aoRBC/l7SLJ5Bx3TdJu+LKtjGHSvmxOt1Sm5xzG2H3KEO80G0w4R4j+J8UH4uLPqMkXMnXh36dO8fdSao/wB9W7+GJt7ZTGY5wy2UDueuqQaQktJ0LECxIYliQGNOORKNQsBX2rUH3rKZi/WqCuUGCe5OIVHnxJnlInzCQjOxZJE+9AUmJ8yyGkZBzdVTdK6ZKDtSWSenqdknFdEt1H3bSmGbFVcU8+emT/HXbX0n1p3LHuDu4fuj1HCsLs+CNrbzy8X56ZyTx01kjmDm6V5u4+MCCJIYFiTHtXPAguvtDyt5eQvGqTynZUYlGBF9KmIwRaQSunEjSpK85N0t2asId9bE38tUER8E2iFQ76z1+JyBOXC3lntNI69HgCcZ2HN/BwvULWXtwLYM6DeJHk37E5N6T2ZB3jIdey2XzwSCXDOrKI3NGMqhHeuPb0gwsSGJYkBjTsUSPH6HklRcpXraCss37QSGxU4iM8yoIjAzgH3GRcy9L1hTny77uZbHhKicgYi+vPZwLZQW1ZdJ7nxwWPUdC18ENnjzsTFSVd/a+w6INi9hfup+rs67mvgn30Tu1Ly+v3cOiN7dRXh3hO5cO4O6rB5OW2DrdXRYkMSxIjOm4woWFBJcuofjVv1G5ZScIpPSCQGYRgX4Vzkn6fhOdGyaL3cttj24/+Ua+7sOgV0zXVI+RkBr/oeGrIlX8IfcPPPv5s0SiEeaOnMsto2+hosrLr5Zv4y8b9tEjPZEfzxzONeP6tHh3lwVJDAsSY84N1Xv21M6hkpeH+LykDetGIKuUtLQv8HTOPLVbqstA8LbuCe7DZYd5/OPHWbprKT1SenDv+HuZNWAWG/OLeei1TXyWX8ykAV2Yf81IhvWqf2TmeLAgiWFBYsy5RVWp3JRLcEkOxcuWESk4iiQn401PR3w+xO8Hvw/x+Z3X7gO/s+6k5X4f+Nyyfn+dZTHb8PtPLuuWE5+7rGa7/pj91ZRNcMrnFm3lic8Wk1u0jZE9xzJvygMM6z6av27Yx6+WbyVYGebbU87j3q8NISP59HPINBcLkhgWJMacuzQSoWzNGkrff59oeTmEwmjYfYRCaDgE4TAauzwcRkPVJ5d1H4RC7vvCzg2XcRYV8PgTEJ+PShXKo4J6vaSlJpGamozH74M6gXgi0PwJ9PrpT/D37t2ofTc0SNrOBcvGGBMH4vWSdsklpF1ySbNvWyMRN3TCEA7FhFNNMLkhVfOodtfFLg/VBpeGwydCrbKqjA3715B7aCOJKlzYZSRDA4M4Hqzk410FFJVU0D3By5ieKaT7pHa7oTDR8oraukSizd7uuixIjDGmkcTrdaYZTkyMy/b7chdji/NYtGERL+avpH/gOPMmzuOmPpfxPx/ns3D5VgrLqrlpchb3TR1Kp5TWuVHTuraMMaYdWJW/il+t/xV5wTwu63sZ8ybOo0tCJo+/tZ0/rM4jI9nP/dOHccOEfng8zXN1l50jiWFBYozpCEKREC9vfZmnNz5NZbiSm4bfxPfGfo/8QuXh13JZl3eMsZkZPHLNKMb169Tk/VmQxLAgMcZ0JEcrjrL4k8W8uuNVOid15p4L72HO+XNY8tkhfrl0CwWlVdwwoR/zpg2la1rju90sSGJYkBhjOqLcwlwWrF3ApwWfMqLrCB6c9CDnB0byf/+xg999mEdKgpeXb53CqL6Nm7/EgiSGBYkxpqNSVZbtXsZjHz3GkfIjzBwwk3vH30tJaSrPrtrFL64dTYKvcXOaWJDEsCAxxnR05aFynvv8OX6f+3u8Hi+3jL6FuSPnkuiNf9dW6029ZYwxptmk+FO4+8K7ee3a17ikzyUs/mQx1/z9GrYf3x73fVuQGGNMB5KZnsnjX3mc56Y+R/9AfzLTMuO+T7sh0RhjOqDJvSczuffkFtlXXI9IRGS6iGwTkZ0i8qN61t8sIgUi8qn7uCVm3UIR2eQ+bohZ/kd3m5tE5AURif/IZcYYY04rbkEiIl7gSWAGMAK4UURG1FP0L6o6zn085753FnAhMA6YDMwTkZqxk/8IDANGA8nALfVs0xhjTAuJ5xHJJGCnqu5S1Wrgz8A1DXzvCOB9VQ2rahmwEZgOoKrL1AWsA+LfAWiMMea04hkkfYF9Ma/z3WV1/auIfCYifxORmsmWNwIzRCRFRLoBXwFOmojZ7dL6NrC8vp2LyG0iskFENhQUFNRXxBhjTDOIZ5DUN2pY3ZtWcoD+qjoGeBv4PYCqrgCWAf8E/gSsBsJ13vtfwEpVXVXfzlX1GVWdoKoTunfv3vhWGGOMOaN4Bkk+Jx9FZAIHYguoaqGqVrkvnwXGx6z7pXve5Gs4obSjZp2IPAx0B34Qp7obY4xpoHgGyXpgsIgMEJEE4BvA67EFRCR22q45wBZ3uVdEurrPxwBjgBXu61uAacCNqhr/GVuMMcacUdzuI1HVsIjcCbwJeIEXVDVXROYDG1T1deBuEZmD0211DLjZfbsfWCUiAEHgW6pa07X1NLAHWO2uf0VV58erHcYYY87snBhrS0QKcMKnMboBR5uxOu2BtfncYG3u+Jra3vNU9awnmc+JIGkKEdnQkEHLOhJr87nB2tzxtVR7bawtY4wxTWJBYowxpkksSM7umdauQCuwNp8brM0dX4u0186RGGOMaRI7IjHGGNMkFiSuBgx5f7uIfO4Od//BaUYyblfO1uaYcteJiIpIu77apSnTGrRXDfkdi8j1IrJZRHJF5OWWrmNza8Dv+fGY3/F2ESlqjXo2pwa0OUtE3hWRT9yxDWc2awVU9Zx/4Nww+QUwEEjAGTRyRJ0ygZjnc4DlrV3veLfZLZcOrATWABNau95x/h3fDPxna9e1hds8GPgE6Oy+7tHa9Y53m+uUvwvnZulWr3ucf8/PAN93n48A8pqzDnZE4jjrkPeqGox5mcqpA1C2Nw0d5v/nwK+AypasXBw0ZVqD9qohbb4VeFJVjwOo6pEWrmNz+7K/5xtxBoZtzxrSZgVq5nTKoM64h01lQeJo0JD3InKHiHyB88V6dwvVLV7O2mYRuQDop6pLWrJicdKUaQ3aq4a0eQgwREQ+FJE1IjK9xWoXHw39PSMi5wEDgHdaoF7x1JA2/wz4lojk44ysfldzVsCCxNGQIe9R1SdV9XzgAeCnca9VfJ2xzSLiAR4HfthiNYqvRk9r0I41pM0+nO6tK3H+On9ORDrFuV7x1KDPsusbwN9UNRLH+rSEhrT5RuBFVc0EZgIvuZ/xZmFB4jjrkPd1/Bm4Nq41ir+ztTkdGAW8JyJ5wBTg9XZ8wr1J0xq0Uw35f50PvKaqIVXdDWzDCZb26st8lr9B++/Wgoa1+bvAXwFUdTWQhDMOV7OwIHE0ZMj72A/XLGLmR2mnzthmVS1W1W6q2l9V++OcbJ+jqhtap7pN1uhpDdqxs7YZ+DvODKS4s5EOAXa1aC2bV0PajIgMBTrjTJrX3jWkzXuBqwFEZDhOkDTb1LFxG0a+PdGGDXl/p4h8FQgBx4G5rVfjpmtgmzuMBrb3dNMatEsNbPObwFQR2QxEgHmqWth6tW6aL/H/+kbgz+pextSeNbDNPwSeFZF7cbq9bm7Ottud7cYYY5rEuraMMcY0iQWJMcaYJrEgMcYY0yQWJMYYY5rEgsQYY0yTWJAY00Qi8jMRua8N1CPPvRfEmBZlQWKMMaZJLEiMqYeIpIrIUhHZKCKbROSG2L/4RWSCiLwX85axIvKOiOwQkVvdMr1FZKU778UmEbnMXf6UiGxw5/94JGafeSLyf0Rktbv+QhF5U0S+EJHb3TJXutt81Z1D5On6xkwSkW+JyDp3378VEW88f17m3GZBYkz9pgMHVHWsqo4Clp+l/BicoXMuAh4SkT7AN4E3VXUcMBb41C37E1Wd4L7nChEZE7Odfap6EbAKeBG4Dmecs/kxZSbh3Kk8Gjgf+F+xFXGHwLgBuMTddwS46Uu03ZgvxYZIMaZ+nwP/ISILgSWqukqkvkFWT3hNVSuAChF5F+fLfj3wgoj4gb+rak2QXC8it+F8/nrjTDT0mbuuZgiPz4E0VS0BSkSkMmZU3nWqugtARP4EXAr8LaYuV+MMOLnerXMy0N7nGTFtmAWJMfVQ1e0iMh5nyO1HRWQFzhhcNUfxSXXfcuomdKWIXI5zpPKSiCzCOdK4D5ioqsdF5MU626oZfTga87zmdc3n9ZR91XktwO9V9cGzNNOYZmFdW8bUw+2aKlfV/wb+A7gQyKN2aPl/rfOWa0QkSUS64sztsd6dOOmIqj4LPO9uIwCUAcUi0hOY0YjqTXJHevXgdGF9UGf9P4DrRKSH25Yubl2MiQs7IjGmfqOBRSISxRnx+fs4XUTPi8iPgbV1yq8DlgJZwM9V9YCIzAXmiUgIKAX+TVV3i8gnQC7OcO0fNqJuq4EFbh1XAq/GrlTVzSLyU2CFGzYh4A5gTyP2ZcxZ2ei/xrQjInIlcJ+qZrd2XYypYV1bxhhjmsSOSIwxxjSJHZEYY4xpEgsSY4wxTWJBYowxpkksSIwxxjSJBYkxxpgmsSAxxhjTJP8f7/ucJQS6n6cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (gsearch3_1.best_score_, gsearch3_1.best_params_))\n",
    "test_means = gsearch3_1.cv_results_[ 'mean_test_score' ]\n",
    "test_stds = gsearch3_1.cv_results_[ 'std_test_score' ]\n",
    "train_means = gsearch3_1.cv_results_[ 'mean_train_score' ]\n",
    "train_stds = gsearch3_1.cv_results_[ 'std_train_score' ]\n",
    "\n",
    "pd.DataFrame(gsearch3_1.cv_results_).to_csv('my_preds_subsampleh_colsample_bytree_1.csv')\n",
    "\n",
    "# plot results\n",
    "test_scores = np.array(test_means).reshape(len(colsample_bytree), len(subsample))\n",
    "train_scores = np.array(train_means).reshape(len(colsample_bytree), len(subsample))\n",
    "\n",
    "for i, value in enumerate(colsample_bytree):\n",
    "    pyplot.plot(subsample, -test_scores[i], label= 'test_colsample_bytree:'   + str(value))\n",
    "#for i, value in enumerate(min_child_weight):\n",
    "#    pyplot.plot(max_depth, train_scores[i], label= 'train_min_child_weight:'   + str(value))\n",
    "    \n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'subsample' )                                                                                                      \n",
    "pyplot.ylabel( 'Log Loss' )\n",
    "pyplot.savefig( 'subsample_vs_colsample_bytree1.png' )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
